From pypy.commits at gmail.com  Sun Jul  1 01:06:12 2018
From: pypy.commits at gmail.com (mattip)
Date: Sat, 30 Jun 2018 22:06:12 -0700 (PDT)
Subject: [pypy-commit] pypy unicode-utf8-py3: pass more module/_io tests
Message-ID: <5b386144.1c69fb81.410a0.1311@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: unicode-utf8-py3
Changeset: r94792:63fcd825e9f2
Date: 2018-07-01 00:05 -0500
http://bitbucket.org/pypy/pypy/changeset/63fcd825e9f2/

Log:	pass more module/_io tests

diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -1715,6 +1715,9 @@
     def convert_to_w_unicode(self, w_obj):
         return w_obj.convert_to_w_unicode(self)
 
+    def realunicode_w(self, w_obj):
+        return w_obj.utf8_w(self).decode('utf8')
+
     def utf8_0_w(self, w_obj):
         "Like utf8_w, but rejects strings with NUL bytes."
         from rpython.rlib import rstring
diff --git a/pypy/module/_io/interp_textio.py b/pypy/module/_io/interp_textio.py
--- a/pypy/module/_io/interp_textio.py
+++ b/pypy/module/_io/interp_textio.py
@@ -739,7 +739,7 @@
 
     def _read(self, space, size):
         remaining = size
-        builder = UnicodeBuilder(size)
+        builder = StringBuilder(size)
 
         # Keep reading chunks until we have n characters to return
         while remaining > 0:
@@ -825,7 +825,7 @@
 
         result = builder.build()
         lgt = get_utf8_length(result)
-        return space.newutf8(result, lgt)
+        return (result, lgt, lgt)
 
     # _____________________________________________________________
     # write methods
diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py
--- a/pypy/objspace/std/objspace.py
+++ b/pypy/objspace/std/objspace.py
@@ -387,11 +387,11 @@
         if isinstance(s, unicode):
             s, lgt = s.encode('utf8'), len(s)
         elif isinstance(s, str):
-            s, lgt, chk = str_decode_utf8(s, "string", True, None,
+            s, uf8lgt, lgt = str_decode_utf8(s, "string", True, None,
                                            allow_surrogates=True)
         elif isinstance(s, tuple):
             # result of decode_utf8
-            s, lgt, chk = s
+            s, utf8lgt, lgt = s
         else:
             # XXX what is s ?
             lgt = rutf8.check_utf8(s, True)
diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -587,7 +587,7 @@
         return space.newbool(cased)
 
     def descr_isidentifier(self, space):
-        return space.newbool(_isidentifier(self._value))
+        return space.newbool(_isidentifier(self._utf8.decode('utf8')))
 
     def descr_startswith(self, space, w_prefix, w_start=None, w_end=None):
         start, end = self._unwrap_and_compute_idx_params(space, w_start, w_end)

From pypy.commits at gmail.com  Sun Jul  1 16:51:16 2018
From: pypy.commits at gmail.com (arigo)
Date: Sun, 01 Jul 2018 13:51:16 -0700 (PDT)
Subject: [pypy-commit] pypy default: add test that passes on pypy2 but not
 pypy3
Message-ID: <5b393ec4.1c69fb81.4715f.7ebf@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r94793:55061f499bd5
Date: 2018-07-01 22:39 +0200
http://bitbucket.org/pypy/pypy/changeset/55061f499bd5/

Log:	add test that passes on pypy2 but not pypy3

diff --git a/pypy/interpreter/test/test_raise.py b/pypy/interpreter/test/test_raise.py
--- a/pypy/interpreter/test/test_raise.py
+++ b/pypy/interpreter/test/test_raise.py
@@ -280,3 +280,15 @@
             def __new__(cls, *args):
                 return object()
         raises(TypeError, "raise MyException")
+
+    def test_with_exit_True(self):
+        class X:
+            def __enter__(self):
+                pass
+            def __exit__(self, *args):
+                return True
+        def g():
+            with X():
+                return 42
+            assert False, "unreachable"
+        assert g() == 42

From pypy.commits at gmail.com  Sun Jul  1 16:51:18 2018
From: pypy.commits at gmail.com (arigo)
Date: Sun, 01 Jul 2018 13:51:18 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: hg merge default
Message-ID: <5b393ec6.1c69fb81.529e.1a10@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r94794:bd8889a5d67f
Date: 2018-07-01 22:40 +0200
http://bitbucket.org/pypy/pypy/changeset/bd8889a5d67f/

Log:	hg merge default

diff --git a/pypy/interpreter/test/test_raise.py b/pypy/interpreter/test/test_raise.py
--- a/pypy/interpreter/test/test_raise.py
+++ b/pypy/interpreter/test/test_raise.py
@@ -309,6 +309,17 @@
                 return object()
         raises(TypeError, "raise MyException")
 
+    def test_with_exit_True(self):
+        class X:
+            def __enter__(self):
+                pass
+            def __exit__(self, *args):
+                return True
+        def g():
+            with X():
+                return 42
+            assert False, "unreachable"
+        assert g() == 42
 
     def test_pop_exception_value(self):
         # assert that this code don't crash

From pypy.commits at gmail.com  Sun Jul  1 16:51:21 2018
From: pypy.commits at gmail.com (arigo)
Date: Sun, 01 Jul 2018 13:51:21 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Add another test for 'async with',
 and fix for 'with' or 'async with':
Message-ID: <5b393ec9.1c69fb81.148c1.1112@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r94795:0a4016e8a6bc
Date: 2018-07-01 22:50 +0200
http://bitbucket.org/pypy/pypy/changeset/0a4016e8a6bc/

Log:	Add another test for 'async with', and fix for 'with' or 'async
	with': if they call '__exit__' or '__aexit__' and this returns True,
	we must ignore that if we're processing a return/break/continue
	instead of a real exception.

diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py
--- a/pypy/interpreter/pyopcode.py
+++ b/pypy/interpreter/pyopcode.py
@@ -1260,9 +1260,11 @@
 
     def WITH_CLEANUP_FINISH(self, oparg, next_instr):
         w_suppress = self.popvalue()
-        if self.space.is_true(w_suppress):
-            # __exit__() returned True -> Swallow the exception.
-            self.settopvalue(self.space.w_None)
+        w_unroller = self.peekvalue()
+        if isinstance(w_unroller, SApplicationException):
+            if self.space.is_true(w_suppress):
+                # __exit__() returned True -> Swallow the exception.
+                self.settopvalue(self.space.w_None)
         # this is always followed by END_FINALLY
         # in the stack now: [w_unroller-or-w_None..]
 
diff --git a/pypy/interpreter/test/test_coroutine.py b/pypy/interpreter/test/test_coroutine.py
--- a/pypy/interpreter/test/test_coroutine.py
+++ b/pypy/interpreter/test/test_coroutine.py
@@ -112,6 +112,27 @@
         assert seen == ['aenter', 'aexit']
         """
 
+    def test_async_with_exit_True(self): """
+        seen = []
+        class X:
+            async def __aenter__(self):
+                seen.append('aenter')
+            async def __aexit__(self, *args):
+                seen.append('aexit')
+                return True
+        async def f(x):
+            async with x:
+                return 42
+        c = f(X())
+        try:
+            c.send(None)
+        except StopIteration as e:
+            assert e.value == 42
+        else:
+            assert False, "should have raised"
+        assert seen == ['aenter', 'aexit']
+        """
+
     def test_await(self): """
         class X:
             def __await__(self):

From pypy.commits at gmail.com  Mon Jul  2 00:12:55 2018
From: pypy.commits at gmail.com (mattip)
Date: Sun, 01 Jul 2018 21:12:55 -0700 (PDT)
Subject: [pypy-commit] pypy unicode-utf8-py3: fix interp_stringio for utf8
Message-ID: <5b39a647.1c69fb81.529e.8dd3@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: unicode-utf8-py3
Changeset: r94796:9613305bf7cb
Date: 2018-07-01 11:12 -0500
http://bitbucket.org/pypy/pypy/changeset/9613305bf7cb/

Log:	fix interp_stringio for utf8

diff --git a/pypy/module/_io/interp_stringio.py b/pypy/module/_io/interp_stringio.py
--- a/pypy/module/_io/interp_stringio.py
+++ b/pypy/module/_io/interp_stringio.py
@@ -46,7 +46,7 @@
         limit = self._convert_limit(limit)
         start = self.pos
         if start >= len(self.data):
-            return u''
+            return ''
         end = start + limit
         pos = start
         while pos < end:
@@ -70,7 +70,7 @@
         start = self.pos
         limit = self._convert_limit(limit)
         if start >= len(self.data):
-            return u''
+            return ''
         end = start + limit
         found = False
         for pos in range(start, end - len(marker) + 1):

From pypy.commits at gmail.com  Mon Jul  2 00:12:58 2018
From: pypy.commits at gmail.com (mattip)
Date: Sun, 01 Jul 2018 21:12:58 -0700 (PDT)
Subject: [pypy-commit] pypy unicode-utf8-py3: fixes from testing module/_ast
Message-ID: <5b39a64a.1c69fb81.c3f2.35a0@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: unicode-utf8-py3
Changeset: r94797:599273325eea
Date: 2018-07-01 16:37 -0500
http://bitbucket.org/pypy/pypy/changeset/599273325eea/

Log:	fixes from testing module/_ast

diff --git a/pypy/objspace/std/bytesobject.py b/pypy/objspace/std/bytesobject.py
--- a/pypy/objspace/std/bytesobject.py
+++ b/pypy/objspace/std/bytesobject.py
@@ -14,6 +14,8 @@
 from pypy.objspace.std.stringmethods import StringMethods
 from pypy.objspace.std.util import IDTAG_SPECIAL, IDTAG_SHIFT
 from pypy.objspace.std.formatting import mod_format, FORMAT_BYTES
+from pypy.objspace.std.unicodeobject import (encode_object, getdefaultencoding,
+           decode_object)
 
 class W_AbstractBytesObject(W_Root):
     __slots__ = ()
@@ -688,7 +690,6 @@
             raise oefmt(space.w_TypeError,
                 "encoding without string argument (got '%T' instead)",
                 w_source)
-        from pypy.objspace.std.unicodeobject import encode_object
         w_source = encode_object(space, w_source, encoding, errors)
         # and continue with the encoded string
     elif errors is not None:
diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -1144,14 +1144,13 @@
 
 
 def encode_object(space, w_object, encoding, errors):
-    w_encoder = None
-    if encoding is None:
-        # Get the encoder functions as a wrapped object.
-        # This lookup is cached.
-        w_encoder = space.sys.get_w_default_encoder()
     if errors is None or errors == 'strict':
-        if ((encoding is None and space.sys.defaultencoding == 'ascii') or
-             encoding == 'ascii'):
+        if encoding is None or encoding == 'utf-8':
+            utf8 = space.utf8_w(w_object)
+            if rutf8.has_surrogates(utf8):
+                utf8 = rutf8.reencode_utf8_with_surrogates(utf8)
+            return space.newbytes(utf8)
+        elif encoding == 'ascii':
             s = space.utf8_w(w_object)
             try:
                 rutf8.check_ascii(s)
@@ -1161,21 +1160,11 @@
                     a.pos, a.pos + 1)
                 assert False, "always raises"
             return space.newbytes(s)
-        if ((encoding is None and space.sys.defaultencoding == 'utf8') or
-             encoding == 'utf-8' or encoding == 'utf8' or encoding == 'UTF-8'):
-            utf8 = space.utf8_w(w_object)
-            if rutf8.has_surrogates(utf8):
-                utf8 = rutf8.reencode_utf8_with_surrogates(utf8)
-            return space.newbytes(utf8)
-    if w_encoder is None:
-        from pypy.module._codecs.interp_codecs import lookup_codec
-        w_encoder = space.getitem(lookup_codec(space, encoding), space.newint(0))
-    if errors is None:
-        w_errors = space.newtext('strict')
-    else:
-        w_errors = space.newtext(errors)
-    w_restuple = space.call_function(w_encoder, w_object, w_errors)
-    w_retval = space.getitem(w_restuple, space.newint(0))
+
+    from pypy.module._codecs.interp_codecs import encode_text
+    if encoding is None:
+        encoding = space.sys.defaultencoding
+    w_retval = encode_text(space, w_object, encoding, errors)
     if not space.isinstance_w(w_retval, space.w_bytes):
         raise oefmt(space.w_TypeError,
                     "'%s' encoder returned '%T' instead of 'bytes'; "

From pypy.commits at gmail.com  Mon Jul  2 00:13:00 2018
From: pypy.commits at gmail.com (mattip)
Date: Sun, 01 Jul 2018 21:13:00 -0700 (PDT)
Subject: [pypy-commit] pypy unicode-utf8-py3: fixes from module/_sre/test
Message-ID: <5b39a64c.1c69fb81.239b6.fd8e@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: unicode-utf8-py3
Changeset: r94798:ac5381b72782
Date: 2018-07-01 22:41 -0500
http://bitbucket.org/pypy/pypy/changeset/ac5381b72782/

Log:	fixes from module/_sre/test

diff --git a/pypy/module/_sre/interp_sre.py b/pypy/module/_sre/interp_sre.py
--- a/pypy/module/_sre/interp_sre.py
+++ b/pypy/module/_sre/interp_sre.py
@@ -163,8 +163,8 @@
         string = None
         buf = None
         space = self.space
-        if space.isinstance_w(w_string, space.w_utf8):
-            unicodestr = space.utf8_w(w_string).decode()
+        if space.isinstance_w(w_string, space.w_unicode):
+            unicodestr = space.utf8_w(w_string).decode('utf8')
             length = len(unicodestr)
         elif space.isinstance_w(w_string, space.w_bytes):
             string = space.bytes_w(w_string)
@@ -176,6 +176,7 @@
         return (length, unicodestr, string, buf)
 
     def make_ctx(self, w_string, pos=0, endpos=sys.maxint, flags=0):
+        """Make a StrMatchContext, BufMatchContext or a Utf8MatchContext for
         searching in the given w_string object."""
         space = self.space
         length, unicodestr, string, buf = self.getstring(w_string)
diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py
--- a/pypy/objspace/std/objspace.py
+++ b/pypy/objspace/std/objspace.py
@@ -553,7 +553,7 @@
             return w_obj.listview_utf8()
         if type(w_obj) is W_SetObject or type(w_obj) is W_FrozensetObject:
             return w_obj.listview_utf8()
-        if (isinstance(w_obj, W_UnicodeObject) and self._uni_uses_no_iter(w_obj)
+        if (isinstance(w_obj, W_UnicodeObject) and not self._uses_unicode_iter(w_obj)
             and w_obj.is_ascii()):
             return w_obj.listview_utf8()
         if isinstance(w_obj, W_ListObject) and self._uses_list_iter(w_obj):

From pypy.commits at gmail.com  Mon Jul  2 00:13:02 2018
From: pypy.commits at gmail.com (mattip)
Date: Sun, 01 Jul 2018 21:13:02 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6: merge py3.5 into py3.6
Message-ID: <5b39a64e.1c69fb81.25e59.0e58@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: py3.6
Changeset: r94799:289d8154d34d
Date: 2018-07-01 22:42 -0500
http://bitbucket.org/pypy/pypy/changeset/289d8154d34d/

Log:	merge py3.5 into py3.6

diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py
--- a/pypy/interpreter/pyopcode.py
+++ b/pypy/interpreter/pyopcode.py
@@ -1287,9 +1287,11 @@
 
     def WITH_CLEANUP_FINISH(self, oparg, next_instr):
         w_suppress = self.popvalue()
-        if self.space.is_true(w_suppress):
-            # __exit__() returned True -> Swallow the exception.
-            self.settopvalue(self.space.w_None)
+        w_unroller = self.peekvalue()
+        if isinstance(w_unroller, SApplicationException):
+            if self.space.is_true(w_suppress):
+                # __exit__() returned True -> Swallow the exception.
+                self.settopvalue(self.space.w_None)
         # this is always followed by END_FINALLY
         # in the stack now: [w_unroller-or-w_None..]
 
diff --git a/pypy/interpreter/test/test_coroutine.py b/pypy/interpreter/test/test_coroutine.py
--- a/pypy/interpreter/test/test_coroutine.py
+++ b/pypy/interpreter/test/test_coroutine.py
@@ -130,6 +130,27 @@
         assert seen == ['aenter', 'aexit']
         """
 
+    def test_async_with_exit_True(self): """
+        seen = []
+        class X:
+            async def __aenter__(self):
+                seen.append('aenter')
+            async def __aexit__(self, *args):
+                seen.append('aexit')
+                return True
+        async def f(x):
+            async with x:
+                return 42
+        c = f(X())
+        try:
+            c.send(None)
+        except StopIteration as e:
+            assert e.value == 42
+        else:
+            assert False, "should have raised"
+        assert seen == ['aenter', 'aexit']
+        """
+
     def test_await(self): """
         class X:
             def __await__(self):
diff --git a/pypy/interpreter/test/test_raise.py b/pypy/interpreter/test/test_raise.py
--- a/pypy/interpreter/test/test_raise.py
+++ b/pypy/interpreter/test/test_raise.py
@@ -309,6 +309,17 @@
                 return object()
         raises(TypeError, "raise MyException")
 
+    def test_with_exit_True(self):
+        class X:
+            def __enter__(self):
+                pass
+            def __exit__(self, *args):
+                return True
+        def g():
+            with X():
+                return 42
+            assert False, "unreachable"
+        assert g() == 42
 
     def test_pop_exception_value(self):
         # assert that this code don't crash

From pypy.commits at gmail.com  Mon Jul  2 00:13:04 2018
From: pypy.commits at gmail.com (mattip)
Date: Sun, 01 Jul 2018 21:13:04 -0700 (PDT)
Subject: [pypy-commit] pypy unicode-utf8-py3: merge py3.5 into branch
Message-ID: <5b39a650.1c69fb81.32140.6fa7@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: unicode-utf8-py3
Changeset: r94800:ebb1b24a3065
Date: 2018-07-01 22:43 -0500
http://bitbucket.org/pypy/pypy/changeset/ebb1b24a3065/

Log:	merge py3.5 into branch

diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py
--- a/pypy/interpreter/pyopcode.py
+++ b/pypy/interpreter/pyopcode.py
@@ -1260,9 +1260,11 @@
 
     def WITH_CLEANUP_FINISH(self, oparg, next_instr):
         w_suppress = self.popvalue()
-        if self.space.is_true(w_suppress):
-            # __exit__() returned True -> Swallow the exception.
-            self.settopvalue(self.space.w_None)
+        w_unroller = self.peekvalue()
+        if isinstance(w_unroller, SApplicationException):
+            if self.space.is_true(w_suppress):
+                # __exit__() returned True -> Swallow the exception.
+                self.settopvalue(self.space.w_None)
         # this is always followed by END_FINALLY
         # in the stack now: [w_unroller-or-w_None..]
 
diff --git a/pypy/interpreter/test/test_coroutine.py b/pypy/interpreter/test/test_coroutine.py
--- a/pypy/interpreter/test/test_coroutine.py
+++ b/pypy/interpreter/test/test_coroutine.py
@@ -112,6 +112,27 @@
         assert seen == ['aenter', 'aexit']
         """
 
+    def test_async_with_exit_True(self): """
+        seen = []
+        class X:
+            async def __aenter__(self):
+                seen.append('aenter')
+            async def __aexit__(self, *args):
+                seen.append('aexit')
+                return True
+        async def f(x):
+            async with x:
+                return 42
+        c = f(X())
+        try:
+            c.send(None)
+        except StopIteration as e:
+            assert e.value == 42
+        else:
+            assert False, "should have raised"
+        assert seen == ['aenter', 'aexit']
+        """
+
     def test_await(self): """
         class X:
             def __await__(self):
diff --git a/pypy/interpreter/test/test_raise.py b/pypy/interpreter/test/test_raise.py
--- a/pypy/interpreter/test/test_raise.py
+++ b/pypy/interpreter/test/test_raise.py
@@ -309,6 +309,17 @@
                 return object()
         raises(TypeError, "raise MyException")
 
+    def test_with_exit_True(self):
+        class X:
+            def __enter__(self):
+                pass
+            def __exit__(self, *args):
+                return True
+        def g():
+            with X():
+                return 42
+            assert False, "unreachable"
+        assert g() == 42
 
     def test_pop_exception_value(self):
         # assert that this code don't crash

From pypy.commits at gmail.com  Mon Jul  2 00:13:06 2018
From: pypy.commits at gmail.com (mattip)
Date: Sun, 01 Jul 2018 21:13:06 -0700 (PDT)
Subject: [pypy-commit] pypy unicode-utf8: merged default into branch
Message-ID: <5b39a652.1c69fb81.3720a.0b2f@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: unicode-utf8
Changeset: r94801:f32c591c9e7f
Date: 2018-07-01 22:45 -0500
http://bitbucket.org/pypy/pypy/changeset/f32c591c9e7f/

Log:	merged default into branch

diff --git a/pypy/doc/sandbox.rst b/pypy/doc/sandbox.rst
--- a/pypy/doc/sandbox.rst
+++ b/pypy/doc/sandbox.rst
@@ -3,6 +3,11 @@
 PyPy's sandboxing features
 ==========================
 
+.. warning:: This is not actively maintained. You will likely have to fix
+   some issues yourself, or otherwise play around on your own. We provide
+   this documentation for historical reasions, it will not translate or
+   run on the latest PyPy code base.
+
 Introduction
 ------------
 
diff --git a/pypy/interpreter/pyparser/automata.py b/pypy/interpreter/pyparser/automata.py
--- a/pypy/interpreter/pyparser/automata.py
+++ b/pypy/interpreter/pyparser/automata.py
@@ -23,6 +23,10 @@
 
 ERROR_STATE = chr(255)
 
+# NB: all non-ascii bytes (>= 128) will be turned into 128
+NON_ASCII = chr(128)
+
+
 class DFA:
     # ____________________________________________________________
     def __init__(self, states, accepts, start = 0):
@@ -36,7 +40,10 @@
             for key in state:
                 if key == DEFAULT:
                     continue
-                maximum = max(ord(key), maximum)
+                ordkey = ord(key)
+                if ordkey > 128:
+                    raise ValueError("DFA does not support matching of specific non-ASCII character %r. Use NON_ASCII instead" % key)
+                maximum = max(ordkey, maximum)
         self.max_char = maximum + 1
 
         defaults = []
@@ -72,6 +79,8 @@
         i = pos
         for i in range(pos, len(inVec)):
             item = inVec[i]
+            if ord(item) > 0x80:
+                item = NON_ASCII
             accept = self.accepts[crntState]
             crntState = self._next_state(item, crntState)
             if crntState != ERROR_STATE:
@@ -103,6 +112,8 @@
         i = pos
         for i in range(pos, len(inVec)):
             item = inVec[i]
+            if ord(item) > 0x80:
+                item = NON_ASCII
             accept = self.accepts[crntState]
             if accept:
                 return i
diff --git a/pypy/interpreter/pyparser/test/test_automata.py b/pypy/interpreter/pyparser/test/test_automata.py
--- a/pypy/interpreter/pyparser/test/test_automata.py
+++ b/pypy/interpreter/pyparser/test/test_automata.py
@@ -1,4 +1,7 @@
-from pypy.interpreter.pyparser.automata import DFA, NonGreedyDFA, DEFAULT
+# coding: utf-8
+import pytest
+
+from pypy.interpreter.pyparser.automata import DFA, NonGreedyDFA, DEFAULT, NON_ASCII
 
 def test_states():
     d = DFA([{"\x00": 1}, {"\x01": 0}], [False, True])
@@ -27,3 +30,18 @@
     d = NonGreedyDFA([{"a": 1}, {DEFAULT: 0}], [False, True])
     assert d.recognize("a,a?ab") == 1
     assert d.recognize("c") == -1
+
+def test_nonascii():
+    d = DFA([{"a": 1}, {NON_ASCII: 1}], [False, True])
+    input = u"a&#252;&#252;&#252;&#252;".encode("utf-8")
+    assert d.recognize(input) == len(input)
+    assert d.recognize("c") == -1
+    assert d.recognize("&#252;") == -1
+
+    d = NonGreedyDFA([{NON_ASCII: 0, "b": 1}, {"b": 0}], [False, True])
+    input = u"&#252;&#252;bbbb".encode("utf-8")
+    assert d.recognize(input) == len(u"&#252;&#252;b".encode("utf-8"))
+    assert d.recognize("c") == -1
+
+    pytest.raises(ValueError, DFA, [{"\x81": 2}], [True])
+
diff --git a/pypy/interpreter/test/test_function.py b/pypy/interpreter/test/test_function.py
--- a/pypy/interpreter/test/test_function.py
+++ b/pypy/interpreter/test/test_function.py
@@ -455,6 +455,8 @@
         assert repr(B().f).startswith("<bound method B.f of <")
         assert repr(A().f).endswith(">>")
 
+        assert repr(type(A.f)) == repr(type(A().f)) == "<type 'instancemethod'>"
+
 
     def test_method_call(self):
         class C(object):
diff --git a/pypy/interpreter/test/test_raise.py b/pypy/interpreter/test/test_raise.py
--- a/pypy/interpreter/test/test_raise.py
+++ b/pypy/interpreter/test/test_raise.py
@@ -280,3 +280,15 @@
             def __new__(cls, *args):
                 return object()
         raises(TypeError, "raise MyException")
+
+    def test_with_exit_True(self):
+        class X:
+            def __enter__(self):
+                pass
+            def __exit__(self, *args):
+                return True
+        def g():
+            with X():
+                return 42
+            assert False, "unreachable"
+        assert g() == 42
diff --git a/pypy/interpreter/typedef.py b/pypy/interpreter/typedef.py
--- a/pypy/interpreter/typedef.py
+++ b/pypy/interpreter/typedef.py
@@ -688,7 +688,7 @@
 Function.typedef.acceptable_as_base_class = False
 
 Method.typedef = TypeDef(
-    "method",
+    "instancemethod",
     __doc__ = """instancemethod(function, instance, class)
 
 Create an instance method object.""",
diff --git a/pypy/module/_cppyy/converter.py b/pypy/module/_cppyy/converter.py
--- a/pypy/module/_cppyy/converter.py
+++ b/pypy/module/_cppyy/converter.py
@@ -706,7 +706,7 @@
                     "no overload found matching %s", self.signature)
 
 
-class SmartPointerConverter(TypeConverter):
+class SmartPtrConverter(TypeConverter):
     _immutable_fields = ['typecode', 'smartdecl', 'rawdecl', 'deref']
     typecode    = 'V'
 
@@ -753,7 +753,7 @@
         return interp_cppyy.wrap_cppinstance(space, address,
             self.rawdecl, smartdecl=self.smartdecl, deref=self.deref, do_cast=False)
 
-class SmartPointerPtrConverter(SmartPointerConverter):
+class SmartPtrPtrConverter(SmartPtrConverter):
     typecode    = 'o'
 
     def from_memory(self, space, w_obj, w_pycppclass, offset):
@@ -763,7 +763,7 @@
         self._is_abstract(space)
 
 
-class SmartPointerRefConverter(SmartPointerPtrConverter):
+class SmartPtrRefConverter(SmartPtrPtrConverter):
     typecode    = 'V'
 
 
@@ -804,6 +804,12 @@
     compound = helper.compound(name)
     clean_name = capi.c_resolve_name(space, helper.clean_type(name))
     try:
+        return _converters[clean_name+compound](space, default)
+    except KeyError:
+        pass
+
+    # arrays
+    try:
         # array_index may be negative to indicate no size or no size found
         array_size = helper.array_size(_name)     # uses original arg
         # TODO: using clean_name here drops const (e.g. const char[] will
@@ -823,11 +829,11 @@
         check_smart = capi.c_smartptr_info(space, clean_name)
         if check_smart[0]:
             if compound == '':
-                return SmartPointerConverter(space, clsdecl, check_smart[1], check_smart[2])
+                return SmartPtrConverter(space, clsdecl, check_smart[1], check_smart[2])
             elif compound == '*':
-                return SmartPointerPtrConverter(space, clsdecl, check_smart[1], check_smart[2])
+                return SmartPtrPtrConverter(space, clsdecl, check_smart[1], check_smart[2])
             elif compound == '&':
-                return SmartPointerRefConverter(space, clsdecl, check_smart[1], check_smart[2])
+                return SmartPtrRefConverter(space, clsdecl, check_smart[1], check_smart[2])
             # fall through: can still return smart pointer in non-smart way
 
         # type check for the benefit of the annotator
diff --git a/pypy/module/_cppyy/executor.py b/pypy/module/_cppyy/executor.py
--- a/pypy/module/_cppyy/executor.py
+++ b/pypy/module/_cppyy/executor.py
@@ -26,7 +26,7 @@
 
 NULL = lltype.nullptr(jit_libffi.FFI_TYPE_P.TO)
 
-class FunctionExecutor(object):
+class Executor(object):
     def __init__(self, space, extra):
         pass
 
@@ -43,7 +43,7 @@
         raise FastCallNotPossible
 
 
-class PtrTypeExecutor(FunctionExecutor):
+class PtrTypeExecutor(Executor):
     _immutable_fields_ = ['typecode']
     typecode = 'P'
 
@@ -63,7 +63,7 @@
         return W_ArrayInstance(space, shape, sys.maxint/shape.size, ptrval)
 
 
-class VoidExecutor(FunctionExecutor):
+class VoidExecutor(Executor):
     def cffi_type(self, space):
         state = space.fromcache(ffitypes.State)
         return state.c_void
@@ -96,7 +96,7 @@
     _mixin_ = True
 
     def __init__(self, space, extra):
-        FunctionExecutor.__init__(self, space, extra)
+        Executor.__init__(self, space, extra)
         self.do_assign = False
         self.item = rffi.cast(self.c_type, 0)
 
@@ -124,7 +124,7 @@
             rffi.cast(self.c_ptrtype, rffi.cast(rffi.VOIDPP, result)[0]))
 
 
-class CStringExecutor(FunctionExecutor):
+class CStringExecutor(Executor):
     def execute(self, space, cppmethod, cppthis, num_args, args):
         lresult = capi.c_call_l(space, cppmethod, cppthis, num_args, args)
         ccpresult = rffi.cast(rffi.CCHARP, lresult)
@@ -134,7 +134,7 @@
         return space.newbytes(result)
 
 
-class ConstructorExecutor(FunctionExecutor):
+class ConstructorExecutor(Executor):
     def execute(self, space, cppmethod, cpptype, num_args, args):
         from pypy.module._cppyy import interp_cppyy
         newthis = capi.c_constructor(space, cppmethod, cpptype, num_args, args)
@@ -142,12 +142,12 @@
         return space.newlong(rffi.cast(rffi.LONG, newthis))   # really want ptrdiff_t here
 
 
-class InstanceExecutor(FunctionExecutor):
+class InstanceExecutor(Executor):
     # For return of a C++ instance by pointer: MyClass* func()
     _immutable_fields_ = ['clsdecl']
 
     def __init__(self, space, clsdecl):
-        FunctionExecutor.__init__(self, space, clsdecl)
+        Executor.__init__(self, space, clsdecl)
         self.clsdecl = clsdecl
 
     def _wrap_result(self, space, obj):
@@ -338,7 +338,7 @@
         return _executors['void*'](space, None)  # allow at least passing of the pointer
 
     # currently used until proper lazy instantiation available in interp_cppyy
-    return FunctionExecutor(space, None)
+    return Executor(space, None)
  
 
 _executors["void"]                = VoidExecutor
@@ -374,10 +374,10 @@
     )
 
     for c_type, stub, names in type_info:
-        class BasicExecutor(ffitypes.typeid(c_type), NumericExecutorMixin, FunctionExecutor):
+        class BasicExecutor(ffitypes.typeid(c_type), NumericExecutorMixin, Executor):
             _immutable_ = True
             c_stubcall  = staticmethod(stub)
-        class BasicRefExecutor(ffitypes.typeid(c_type), NumericRefExecutorMixin, FunctionExecutor):
+        class BasicRefExecutor(ffitypes.typeid(c_type), NumericRefExecutorMixin, Executor):
             def cffi_type(self, space):
                 state = space.fromcache(ffitypes.State)
                 return state.c_voidp
diff --git a/pypy/module/_cppyy/ffitypes.py b/pypy/module/_cppyy/ffitypes.py
--- a/pypy/module/_cppyy/ffitypes.py
+++ b/pypy/module/_cppyy/ffitypes.py
@@ -119,7 +119,7 @@
                 value = space.bytes_w(w_value)
             if len(value) != 1:
                 raise oefmt(space.w_ValueError,
-                            "usigned char expected, got string of size %d", len(value))
+                            "unsigned char expected, got string of size %d", len(value))
 
         value = rffi.cast(rffi.CHAR, value[0])
         return value     # turn it into a "char" to the annotator
diff --git a/pypy/module/_cppyy/interp_cppyy.py b/pypy/module/_cppyy/interp_cppyy.py
--- a/pypy/module/_cppyy/interp_cppyy.py
+++ b/pypy/module/_cppyy/interp_cppyy.py
@@ -672,24 +672,33 @@
 
     _mixin_ = True
 
-    def construct_template_args(self, w_args):
+    def construct_template_args(self, w_tpArgs, args_w = None):
         space = self.space
         tmpl_args = ''
-        for i in range(space.len_w(w_args)):
-            w_obj = space.getitem(w_args, space.newint(i))
-            if space.isinstance_w(w_obj, space.w_text):
-                s = space.text_w(w_obj)      # string describing type
-            elif space.isinstance_w(w_obj, space.w_type):
+        for i in range(space.len_w(w_tpArgs)):
+            w_tp = space.getitem(w_tpArgs, space.newint(i))
+            if space.isinstance_w(w_tp, space.w_text):
+                s = space.text_w(w_tp)      # string describing type
+            elif space.isinstance_w(w_tp, space.w_type):
                 try:
                     # cppyy bound types
-                    name = space.getattr(w_obj, space.newtext('__cppname__'))
+                    s = space.text_w(space.getattr(w_tp, space.newtext('__cppname__')))
+                    if args_w:
+                        # try to specialize the type match for the given object
+                        cppinstance = self.space.interp_w(W_CPPInstance, args_w[i])
+                        if cppinstance.flags & INSTANCE_FLAGS_IS_RVALUE:
+                            sugar = "&&"
+                        elif cppinstance.flags & INSTANCE_FLAGS_IS_REF:
+                            sugar = "*"
+                        else:
+                            sugar = "&"
+                        s += sugar
                 except OperationError:
                     # generic python types
-                    name = space.getattr(w_obj, space.newtext('__name__'))
-                s = space.text_w(name)
+                    s = space.text_w(space.getattr(w_tp, space.newtext('__name__')))
             else:
                 # builtin types etc.
-                s = space.text_w(space.str(w_obj))
+                s = space.text_w(space.str(w_tp))
             # map python types -> C++ types
             if s == 'str': s = 'std::string'
             if i != 0: tmpl_args += ', '
@@ -712,23 +721,31 @@
             cppol = W_CPPOverload(space, self.scope, funcs[:], self.flags)
         return cppol
 
-    def instantiation_from_args(self, name, args_w):
+    def instantiate_and_call(self, name, args_w):
         # try to match with run-time instantiations
         for cppol in self.master.overloads.values():
             try:
-                cppol.descr_get(self.w_this, []).call(args_w)
+                return cppol.descr_get(self.w_this, []).call(args_w)
             except Exception:
                 pass    # completely ignore for now; have to see whether errors become confusing
 
         # if all failed, then try to deduce from argument types
         w_types = self.space.newtuple([self.space.type(obj_w) for obj_w in args_w])
-        proto = self.construct_template_args(w_types)
+        proto = self.construct_template_args(w_types, args_w)
         method = self.find_method_template(name, proto)
 
         # only cache result if the name retains the full template
-        if len(method.functions) == 1:
-            fullname = capi.c_method_full_name(self.space, method.functions[0].cppmethod)
-            if 0 <= fullname.rfind('>'):
+        fullname = capi.c_method_full_name(self.space, method.functions[0].cppmethod)
+        if 0 <= fullname.rfind('>'):
+            try:
+                existing = self.master.overloads[fullname]
+                allf = existing.functions + method.functions
+                if isinstance(existing, W_CPPStaticOverload):
+                    cppol = W_CPPStaticOverload(self.space, self.scope, allf, self.flags)
+                else:
+                    cppol = W_CPPOverload(self.space, self.scope, allf, self.flags)
+                self.master.overloads[fullname] = cppol
+            except KeyError:
                 self.master.overloads[fullname] = method
 
         return method.descr_get(self.w_this, []).call(args_w)
@@ -747,9 +764,12 @@
             method = self.master.overloads[fullname]
         except KeyError:
             method = self.find_method_template(fullname)
-
-        # cache result (name is always full templated name)
-        self.master.overloads[fullname] = method
+            # cache result (name is always full templated name)
+            self.master.overloads[fullname] = method
+            # also cache on "official" name (may include default template arguments)
+            c_fullname = capi.c_method_full_name(self.space, method.functions[0].cppmethod)
+            if c_fullname != fullname:
+                self.master.overloads[c_fullname] = method
 
         return method.descr_get(self.w_this, [])
 
@@ -774,6 +794,7 @@
             return self  # unbound, so no new instance needed
         cppol = W_CPPTemplateOverload(self.space, self.name, self.scope, self.functions, self.flags)
         cppol.w_this = w_cppinstance
+        cppol.master = self.master
         return cppol     # bound
 
     @unwrap_spec(args_w='args_w')
@@ -787,7 +808,7 @@
         except Exception:
             pass
 
-        return self.instantiation_from_args(self.name, args_w)
+        return self.instantiate_and_call(self.name, args_w)
 
     @unwrap_spec(args_w='args_w')
     def getitem(self, args_w):
@@ -842,7 +863,7 @@
             pass
 
         # try new instantiation
-        return self.instantiation_from_args(self.name, args_w)
+        return self.instantiate_and_call(self.name, args_w)
 
     @unwrap_spec(args_w='args_w')
     def getitem(self, args_w):
diff --git a/pypy/module/_cppyy/pythonify.py b/pypy/module/_cppyy/pythonify.py
--- a/pypy/module/_cppyy/pythonify.py
+++ b/pypy/module/_cppyy/pythonify.py
@@ -408,16 +408,22 @@
 
     # map push_back -> __iadd__ (generally true for STL)
     if 'push_back' in pyclass.__dict__ and not '__iadd__' in pyclass.__dict__:
-        def __iadd__(self, ll):
-            [self.push_back(x) for x in ll]
-            return self
-        pyclass.__iadd__ = __iadd__
+        if 'reserve' in pyclass.__dict__:
+            def iadd(self, ll):
+                self.reserve(len(ll))
+                for x in ll: self.push_back(x)
+                return self
+        else:
+            def iadd(self, ll):
+                for x in ll: self.push_back(x)
+                return self
+        pyclass.__iadd__ = iadd
 
     # map begin()/end() protocol to iter protocol on STL(-like) classes, but
     # not on vector, which is pythonized in the capi (interp-level; there is
     # also the fallback on the indexed __getitem__, but that is slower)
-    if not 'vector' in name[:11] and \
-            ('begin' in pyclass.__dict__ and 'end' in pyclass.__dict__):
+# TODO:    if not (0 <= name.find('vector') <= 5):
+    if ('begin' in pyclass.__dict__ and 'end' in pyclass.__dict__):
         if _cppyy._scope_byname(name+'::iterator') or \
                 _cppyy._scope_byname(name+'::const_iterator'):
             def __iter__(self):
@@ -430,6 +436,20 @@
             pyclass.__iter__ = __iter__
         # else: rely on numbered iteration
 
+    # add python collection based initializer
+    if 0 <= name.find('vector') <= 5:
+        pyclass.__real_init__ = pyclass.__init__
+        def vector_init(self, *args):
+            if len(args) == 1 and isinstance(args[0], (tuple, list)):
+                ll = args[0]
+                self.__real_init__()
+                self.reserve(len(ll))
+                for item in ll:
+                    self.push_back(item)
+                return
+            return self.__real_init__(*args)
+        pyclass.__init__ = vector_init
+
     # combine __getitem__ and __len__ to make a pythonized __getitem__
     if '__getitem__' in pyclass.__dict__ and '__len__' in pyclass.__dict__:
         pyclass._getitem__unchecked = pyclass.__getitem__
@@ -470,7 +490,13 @@
     for p in pythonizors:
         p(pyclass, name)
 
+cppyyIsInitialized = False
 def _post_import_startup():
+    # run only once (function is explicitly called in testing)
+    global cppyyIsInitialized
+    if cppyyIsInitialized:
+        return
+
     # _cppyy should not be loaded at the module level, as that will trigger a
     # call to space.getbuiltinmodule(), which will cause _cppyy to be loaded
     # at pypy-c startup, rather than on the "import _cppyy" statement
@@ -511,6 +537,9 @@
     # install nullptr as a unique reference
     _cppyy.nullptr = _cppyy._get_nullptr()
 
+    # done
+    cppyyIsInitialized = True
+
 
 # user-defined pythonizations interface
 _pythonizations = {'' : list()}
diff --git a/pypy/module/_cppyy/test/templates.h b/pypy/module/_cppyy/test/templates.h
--- a/pypy/module/_cppyy/test/templates.h
+++ b/pypy/module/_cppyy/test/templates.h
@@ -107,7 +107,7 @@
 }
 
 inline std::string tuplify(std::ostringstream& out) {
-    out.seekp(-2, out.cur); out << ')';
+    out << "NULL)";
     return out.str();
 }
 
diff --git a/pypy/module/_cppyy/test/test_templates.py b/pypy/module/_cppyy/test/test_templates.py
--- a/pypy/module/_cppyy/test/test_templates.py
+++ b/pypy/module/_cppyy/test/test_templates.py
@@ -13,7 +13,7 @@
 
     def setup_class(cls):
         cls.w_test_dct  = cls.space.newtext(test_dct)
-        cls.w_datatypes = cls.space.appexec([], """():
+        cls.w_templates = cls.space.appexec([], """():
             import ctypes, _cppyy
             _cppyy._post_import_startup()
             return ctypes.CDLL(%r, ctypes.RTLD_GLOBAL)""" % (test_dct, ))
@@ -84,10 +84,11 @@
 
         import _cppyy
 
-        s = _cppyy.gbl.std.ostringstream()
-        #s << '('
-        #_cppyy.gbl.SomeNS.tuplify(s, 1, 4., "aap")
-        #assert s.str() == '(1, 4, aap)
+        s = _cppyy.gbl.std.ostringstream('(', _cppyy.gbl.std.ios_base.ate)
+        # Fails; selects void* overload (?!)
+        #s << "("
+        _cppyy.gbl.SomeNS.tuplify(s, 1, 4., "aap")
+        assert s.str() == "(1, 4, aap, NULL)"
 
         _cppyy.gbl.gInterpreter.Declare("""
             template<typename... myTypes>
@@ -133,7 +134,7 @@
         Obj2                  = _cppyy.gbl.AttrTesting.Obj2
         select_template_arg   = _cppyy.gbl.AttrTesting.select_template_arg
 
-       #assert select_template_arg[0, Obj1, Obj2].argument == Obj1
+      # assert select_template_arg[0, Obj1, Obj2].argument == Obj1
         assert select_template_arg[1, Obj1, Obj2].argument == Obj2
         raises(TypeError, select_template_arg.__getitem__, 2, Obj1, Obj2)
 
@@ -169,7 +170,7 @@
 
 
       # TODO: the ref_value property is inaccessible (offset == -1)
-      # assert cppyy.gbl.BaseClassWithStatic["size_t"].ref_value == 42
+      # assert _cppyy.gbl.BaseClassWithStatic["size_t"].ref_value == 42
 
         b1 = _cppyy.gbl.DerivedClassUsingStatic["size_t"](  0)
         b2 = _cppyy.gbl.DerivedClassUsingStatic["size_t"](100)
@@ -179,3 +180,62 @@
 
       # assert b2.ref_value == 42
         assert b2.m_value   == 42
+
+
+class AppTestBOOSTANY:
+    spaceconfig = dict(usemodules=['_cppyy', '_rawffi', 'itertools'])
+
+    def setup_class(cls):
+        cls.w_test_dct  = cls.space.newtext(test_dct)
+        cls.w_templates = cls.space.appexec([], """():
+            import ctypes, _cppyy
+            _cppyy._post_import_startup()""")
+
+    def test01_any_class(self):
+        """Usage of boost::any"""
+
+        import _cppyy
+
+        if not _cppyy.gbl.gInterpreter.Declare('#include "boost/any.hpp"'):
+            import warnings
+            warnings.warn('skipping boost/any testing')
+            return
+
+        assert _cppyy.gbl.boost
+        assert _cppyy.gbl.boost.any
+
+        std, boost = _cppyy.gbl.std, _cppyy.gbl.boost
+
+        assert std.list[boost.any]
+
+        val = boost.any()
+        # test both by-ref and by rvalue
+        v = std.vector[int]()
+        val.__assign__(v)
+        val.__assign__(std.move(std.vector[int](range(100))))
+
+        _cppyy.gbl.gInterpreter.ProcessLine(
+            "namespace _cppyy_internal { auto* stdvectid = &typeid(std::vector<int>); }")
+
+        assert val.type() == _cppyy.gbl._cppyy_internal.stdvectid
+
+        extract = boost.any_cast[std.vector[int]](val)
+        assert type(extract) is std.vector[int]
+        assert len(extract) == 100
+        extract += range(100)
+        assert len(extract) == 200
+
+        val.__assign__(std.move(extract))   # move forced
+
+        # TODO: we hit boost::any_cast<int>(boost::any* operand) instead
+        # of the reference version which raises
+        boost.any_cast.__useffi__ = False
+        try:
+          # raises(Exception, boost.any_cast[int], val)
+            assert not boost.any_cast[int](val)
+        except Exception:
+          # getting here is good, too ...
+            pass
+
+        extract = boost.any_cast[std.vector[int]](val)
+        assert len(extract) == 200
diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -643,7 +643,7 @@
     '_PyTraceMalloc_Track', '_PyTraceMalloc_Untrack', 'PyMem_Malloc',
     'PyObject_Free', 'PyObject_GC_Del', 'PyType_GenericAlloc',
     '_PyObject_New', '_PyObject_NewVar',
-    '_PyObject_GC_New', '_PyObject_GC_NewVar',
+    '_PyObject_GC_Malloc', '_PyObject_GC_New', '_PyObject_GC_NewVar',
     'PyObject_Init', 'PyObject_InitVar', 'PyInt_FromLong',
     'PyTuple_New', '_Py_Dealloc',
 ]
@@ -772,6 +772,9 @@
 # a pointer to PyObject
 PyObjectP = rffi.CArrayPtr(PyObject)
 
+# int *
+INTP_real = rffi.CArrayPtr(rffi.INT_real)
+
 def configure_types():
     for config in (CConfig, CConfig2):
         for name, TYPE in rffi_platform.configure(config).iteritems():
diff --git a/pypy/module/cpyext/include/object.h b/pypy/module/cpyext/include/object.h
--- a/pypy/module/cpyext/include/object.h
+++ b/pypy/module/cpyext/include/object.h
@@ -383,6 +383,7 @@
 
 PyAPI_FUNC(PyObject *) _PyObject_New(PyTypeObject *);
 PyAPI_FUNC(PyVarObject *) _PyObject_NewVar(PyTypeObject *, Py_ssize_t);
+PyAPI_FUNC(PyObject *) _PyObject_GC_Malloc(size_t);
 PyAPI_FUNC(PyObject *) _PyObject_GC_New(PyTypeObject *);
 PyAPI_FUNC(PyVarObject *) _PyObject_GC_NewVar(PyTypeObject *, Py_ssize_t);
 
diff --git a/pypy/module/cpyext/longobject.py b/pypy/module/cpyext/longobject.py
--- a/pypy/module/cpyext/longobject.py
+++ b/pypy/module/cpyext/longobject.py
@@ -1,7 +1,7 @@
 from rpython.rtyper.lltypesystem import lltype, rffi
 from pypy.module.cpyext.api import (
     cpython_api, PyObject, build_type_checkers_flags, Py_ssize_t,
-    CONST_STRING, ADDR, CANNOT_FAIL)
+    CONST_STRING, ADDR, CANNOT_FAIL, INTP_real)
 from pypy.objspace.std.longobject import W_LongObject
 from pypy.interpreter.error import OperationError, oefmt
 from pypy.interpreter.unicodehelper import wcharpsize2utf8
@@ -113,7 +113,7 @@
     num = space.bigint_w(w_long)
     return num.ulonglongmask()
 
- at cpython_api([PyObject, rffi.CArrayPtr(rffi.INT_real)], lltype.Signed,
+ at cpython_api([PyObject, INTP_real], lltype.Signed,
              error=-1)
 def PyLong_AsLongAndOverflow(space, w_long, overflow_ptr):
     """
@@ -134,7 +134,7 @@
         overflow_ptr[0] = rffi.cast(rffi.INT_real, -1)
     return -1
 
- at cpython_api([PyObject, rffi.CArrayPtr(rffi.INT_real)], rffi.LONGLONG,
+ at cpython_api([PyObject, INTP_real], rffi.LONGLONG,
              error=-1)
 def PyLong_AsLongLongAndOverflow(space, w_long, overflow_ptr):
     """
diff --git a/pypy/module/cpyext/object.py b/pypy/module/cpyext/object.py
--- a/pypy/module/cpyext/object.py
+++ b/pypy/module/cpyext/object.py
@@ -1,7 +1,7 @@
 from rpython.rtyper.lltypesystem import rffi, lltype
 from pypy.module.cpyext.api import (
     cpython_api, generic_cpy_call, CANNOT_FAIL, Py_ssize_t, Py_ssize_tP,
-    PyVarObject, size_t, slot_function,
+    PyVarObject, size_t, slot_function, INTP_real,
     Py_TPFLAGS_HEAPTYPE, Py_LT, Py_LE, Py_EQ, Py_NE, Py_GT,
     Py_GE, CONST_STRING, FILEP, fwrite, c_only)
 from pypy.module.cpyext.pyobject import (
@@ -221,14 +221,14 @@
     expression cmp(o1, o2)."""
     return space.int_w(space.cmp(w_o1, w_o2))
 
- at cpython_api([PyObject, PyObject, rffi.INTP], rffi.INT_real, error=-1)
+ at cpython_api([PyObject, PyObject, INTP_real], rffi.INT_real, error=-1)
 def PyObject_Cmp(space, w_o1, w_o2, result):
     """Compare the values of o1 and o2 using a routine provided by o1, if one
     exists, otherwise with a routine provided by o2.  The result of the
     comparison is returned in result.  Returns -1 on failure.  This is the
     equivalent of the Python statement result = cmp(o1, o2)."""
     res = space.int_w(space.cmp(w_o1, w_o2))
-    result[0] = rffi.cast(rffi.INT, res)
+    result[0] = rffi.cast(rffi.INT_real, res)
     return 0
 
 @cpython_api([PyObject, PyObject, rffi.INT_real], PyObject)
diff --git a/pypy/module/cpyext/pystrtod.py b/pypy/module/cpyext/pystrtod.py
--- a/pypy/module/cpyext/pystrtod.py
+++ b/pypy/module/cpyext/pystrtod.py
@@ -1,6 +1,6 @@
 import errno
 from pypy.interpreter.error import oefmt
-from pypy.module.cpyext.api import cpython_api, CONST_STRING
+from pypy.module.cpyext.api import cpython_api, CONST_STRING, INTP_real
 from pypy.module.cpyext.pyobject import PyObject
 from rpython.rlib import rdtoa
 from rpython.rlib import rfloat
@@ -80,7 +80,7 @@
         if not user_endptr:
             lltype.free(endptr, flavor='raw')
 
- at cpython_api([rffi.DOUBLE, lltype.Char, rffi.INT_real, rffi.INT_real, rffi.INTP], rffi.CCHARP)
+ at cpython_api([rffi.DOUBLE, lltype.Char, rffi.INT_real, rffi.INT_real, INTP_real], rffi.CCHARP)
 def PyOS_double_to_string(space, val, format_code, precision, flags, ptype):
     """Convert a double val to a string using supplied
     format_code, precision, and flags.
@@ -114,7 +114,7 @@
     buffer, rtype = rfloat.double_to_string(val, format_code,
                                             intmask(precision),
                                             intmask(flags))
-    if ptype != lltype.nullptr(rffi.INTP.TO):
-        ptype[0] = rffi.cast(rffi.INT, DOUBLE_TO_STRING_TYPES_MAP[rtype])
+    if ptype != lltype.nullptr(INTP_real.TO):
+        ptype[0] = rffi.cast(rffi.INT_real, DOUBLE_TO_STRING_TYPES_MAP[rtype])
     bufp = rffi.str2charp(buffer)
     return bufp
diff --git a/pypy/module/cpyext/src/object.c b/pypy/module/cpyext/src/object.c
--- a/pypy/module/cpyext/src/object.c
+++ b/pypy/module/cpyext/src/object.c
@@ -60,6 +60,11 @@
     return (PyObject*)_PyObject_NewVar(type, 0);
 }
 
+PyObject * _PyObject_GC_Malloc(size_t size)
+{
+    return (PyObject *)PyObject_Malloc(size);
+}
+
 PyObject * _PyObject_GC_New(PyTypeObject *type)
 {
     return _PyObject_New(type);
diff --git a/pypy/module/cpyext/test/test_object.py b/pypy/module/cpyext/test/test_object.py
--- a/pypy/module/cpyext/test/test_object.py
+++ b/pypy/module/cpyext/test/test_object.py
@@ -5,7 +5,7 @@
 from rpython.rtyper.lltypesystem import rffi, lltype
 from pypy.module.cpyext.pyobject import get_w_obj_and_decref
 from pypy.module.cpyext.api import (
-    Py_LT, Py_LE, Py_NE, Py_EQ, Py_GE, Py_GT)
+    Py_LT, Py_LE, Py_NE, Py_EQ, Py_GE, Py_GT, INTP_real)
 from pypy.module.cpyext.object import (
     PyObject_IsTrue, PyObject_Not, PyObject_GetAttrString,
     PyObject_DelAttrString, PyObject_GetAttr, PyObject_DelAttr,
@@ -205,7 +205,7 @@
 
     def test_cmp(self, space, api):
         w = space.wrap
-        with lltype.scoped_alloc(rffi.INTP.TO, 1) as ptr:
+        with lltype.scoped_alloc(INTP_real.TO, 1) as ptr:
             assert api.PyObject_Cmp(w(42), w(72), ptr) == 0
             assert ptr[0] == -1
             assert api.PyObject_Cmp(w("a"), w("a"), ptr) == 0
diff --git a/pypy/module/cpyext/test/test_pystrtod.py b/pypy/module/cpyext/test/test_pystrtod.py
--- a/pypy/module/cpyext/test/test_pystrtod.py
+++ b/pypy/module/cpyext/test/test_pystrtod.py
@@ -4,7 +4,7 @@
 from pypy.module.cpyext.test.test_api import BaseApiTest, raises_w
 from rpython.rtyper.lltypesystem import rffi
 from rpython.rtyper.lltypesystem import lltype
-from pypy.module.cpyext.pystrtod import PyOS_string_to_double
+from pypy.module.cpyext.pystrtod import PyOS_string_to_double, INTP_real
 
 
 class TestPyOS_string_to_double(BaseApiTest):
@@ -90,7 +90,7 @@
 class TestPyOS_double_to_string(BaseApiTest):
 
     def test_format_code(self, api):
-        ptype = lltype.malloc(rffi.INTP.TO, 1, flavor='raw')
+        ptype = lltype.malloc(INTP_real.TO, 1, flavor='raw')
         r = api.PyOS_double_to_string(150.0, 'e', 1, 0, ptype)
         assert '1.5e+02' == rffi.charp2str(r)
         type_value = rffi.cast(lltype.Signed, ptype[0])
@@ -99,7 +99,7 @@
         lltype.free(ptype, flavor='raw')
 
     def test_precision(self, api):
-        ptype = lltype.malloc(rffi.INTP.TO, 1, flavor='raw')
+        ptype = lltype.malloc(INTP_real.TO, 1, flavor='raw')
         r = api.PyOS_double_to_string(3.14159269397, 'g', 5, 0, ptype)
         assert '3.1416' == rffi.charp2str(r)
         type_value = rffi.cast(lltype.Signed, ptype[0])
@@ -108,7 +108,7 @@
         lltype.free(ptype, flavor='raw')
 
     def test_flags_sign(self, api):
-        ptype = lltype.malloc(rffi.INTP.TO, 1, flavor='raw')
+        ptype = lltype.malloc(INTP_real.TO, 1, flavor='raw')
         r = api.PyOS_double_to_string(-3.14, 'g', 3, 1, ptype)
         assert '-3.14' == rffi.charp2str(r)
         type_value = rffi.cast(lltype.Signed, ptype[0])
@@ -117,7 +117,7 @@
         lltype.free(ptype, flavor='raw')
 
     def test_flags_add_dot_0(self, api):
-        ptype = lltype.malloc(rffi.INTP.TO, 1, flavor='raw')
+        ptype = lltype.malloc(INTP_real.TO, 1, flavor='raw')
         r = api.PyOS_double_to_string(3, 'g', 5, 2, ptype)
         assert '3.0' == rffi.charp2str(r)
         type_value = rffi.cast(lltype.Signed, ptype[0])
@@ -126,7 +126,7 @@
         lltype.free(ptype, flavor='raw')
 
     def test_flags_alt(self, api):
-        ptype = lltype.malloc(rffi.INTP.TO, 1, flavor='raw')
+        ptype = lltype.malloc(INTP_real.TO, 1, flavor='raw')
         r = api.PyOS_double_to_string(314., 'g', 3, 4, ptype)
         assert '314.' == rffi.charp2str(r)
         type_value = rffi.cast(lltype.Signed, ptype[0])
@@ -135,7 +135,7 @@
         lltype.free(ptype, flavor='raw')
 
     def test_ptype_nan(self, api):
-        ptype = lltype.malloc(rffi.INTP.TO, 1, flavor='raw')
+        ptype = lltype.malloc(INTP_real.TO, 1, flavor='raw')
         r = api.PyOS_double_to_string(float('nan'), 'g', 3, 4, ptype)
         assert 'nan' == rffi.charp2str(r)
         type_value = rffi.cast(lltype.Signed, ptype[0])
@@ -144,7 +144,7 @@
         lltype.free(ptype, flavor='raw')
 
     def test_ptype_infinity(self, api):
-        ptype = lltype.malloc(rffi.INTP.TO, 1, flavor='raw')
+        ptype = lltype.malloc(INTP_real.TO, 1, flavor='raw')
         r = api.PyOS_double_to_string(1e200 * 1e200, 'g', 0, 0, ptype)
         assert 'inf' == rffi.charp2str(r)
         type_value = rffi.cast(lltype.Signed, ptype[0])
@@ -153,8 +153,8 @@
         lltype.free(ptype, flavor='raw')
 
     def test_ptype_null(self, api):
-        ptype = lltype.nullptr(rffi.INTP.TO)
+        ptype = lltype.nullptr(INTP_real.TO)
         r = api.PyOS_double_to_string(3.14, 'g', 3, 0, ptype)
         assert '3.14' == rffi.charp2str(r)
-        assert ptype == lltype.nullptr(rffi.INTP.TO)
+        assert ptype == lltype.nullptr(INTP_real.TO)
         rffi.free_charp(r)
diff --git a/pypy/module/cpyext/test/test_unicodeobject.py b/pypy/module/cpyext/test/test_unicodeobject.py
--- a/pypy/module/cpyext/test/test_unicodeobject.py
+++ b/pypy/module/cpyext/test/test_unicodeobject.py
@@ -4,7 +4,7 @@
 from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
 from pypy.module.cpyext.unicodeobject import (
     Py_UNICODE, PyUnicodeObject, new_empty_unicode)
-from pypy.module.cpyext.api import PyObjectP, PyObject
+from pypy.module.cpyext.api import PyObjectP, PyObject, INTP_real
 from pypy.module.cpyext.pyobject import decref, from_ref
 from rpython.rtyper.lltypesystem import rffi, lltype
 import sys, py
@@ -467,8 +467,8 @@
                     value = 1
                 else:
                     value = 0
-                pendian = lltype.malloc(rffi.INTP.TO, 1, flavor='raw')
-                pendian[0] = rffi.cast(rffi.INT, value)
+                pendian = lltype.malloc(INTP_real.TO, 1, flavor='raw')
+                pendian[0] = rffi.cast(rffi.INT_real, value)
             else:
                 pendian = None
 
@@ -480,7 +480,7 @@
             rffi.free_charp(strict_charp)
             if pendian:
                 if realendian is not None:
-                    assert rffi.cast(rffi.INT, realendian) == pendian[0]
+                    assert rffi.cast(rffi.INT_real, realendian) == pendian[0]
                 lltype.free(pendian, flavor='raw')
 
         test("\x61\x00\x62\x00\x63\x00\x64\x00", -1)
@@ -503,8 +503,8 @@
                     value = 1
                 else:
                     value = 0
-                pendian = lltype.malloc(rffi.INTP.TO, 1, flavor='raw')
-                pendian[0] = rffi.cast(rffi.INT, value)
+                pendian = lltype.malloc(INTP_real.TO, 1, flavor='raw')
+                pendian[0] = rffi.cast(rffi.INT_real, value)
             else:
                 pendian = None
 
diff --git a/pypy/module/cpyext/unicodeobject.py b/pypy/module/cpyext/unicodeobject.py
--- a/pypy/module/cpyext/unicodeobject.py
+++ b/pypy/module/cpyext/unicodeobject.py
@@ -9,7 +9,7 @@
 from pypy.module.unicodedata import unicodedb
 from pypy.module.cpyext.api import (
     CANNOT_FAIL, Py_ssize_t, build_type_checkers_flags, cpython_api,
-    bootstrap_function, CONST_STRING,
+    bootstrap_function, CONST_STRING, INTP_real,
     CONST_WSTRING, slot_function, cts, parse_dir)
 from pypy.module.cpyext.pyerrors import PyErr_BadArgument
 from pypy.module.cpyext.pyobject import (
@@ -532,7 +532,7 @@
 if sys.platform == 'win32':
     make_conversion_functions('MBCS', 'mbcs')
 
- at cpython_api([CONST_STRING, Py_ssize_t, CONST_STRING, rffi.INTP], PyObject)
+ at cpython_api([CONST_STRING, Py_ssize_t, CONST_STRING, INTP_real], PyObject)
 def PyUnicode_DecodeUTF16(space, s, size, llerrors, pbyteorder):
     """Decode length bytes from a UTF-16 encoded buffer string and return the
     corresponding Unicode object.  errors (if non-NULL) defines the error
@@ -579,10 +579,10 @@
     result, _,  length, byteorder = str_decode_utf_16_helper(
         string, errors, final=True, errorhandler=None, byteorder=byteorder)
     if pbyteorder is not None:
-        pbyteorder[0] = rffi.cast(rffi.INT, byteorder)
+        pbyteorder[0] = rffi.cast(rffi.INT_real, byteorder)
     return space.newutf8(result, length)
 
- at cpython_api([CONST_STRING, Py_ssize_t, CONST_STRING, rffi.INTP], PyObject)
+ at cpython_api([CONST_STRING, Py_ssize_t, CONST_STRING, INTP_real], PyObject)
 def PyUnicode_DecodeUTF32(space, s, size, llerrors, pbyteorder):
     """Decode length bytes from a UTF-32 encoded buffer string and
     return the corresponding Unicode object.  errors (if non-NULL)
@@ -631,7 +631,7 @@
     result, _,  length, byteorder = unicodehelper.str_decode_utf_32_helper(
         string, errors, final=True, errorhandler=None, byteorder=byteorder)
     if pbyteorder is not None:
-        pbyteorder[0] = rffi.cast(rffi.INT, byteorder)
+        pbyteorder[0] = rffi.cast(rffi.INT_real, byteorder)
     return space.newutf8(result, length)
 
 @cpython_api([rffi.CWCHARP, Py_ssize_t, rffi.CCHARP, CONST_STRING],

From pypy.commits at gmail.com  Mon Jul  2 00:13:08 2018
From: pypy.commits at gmail.com (mattip)
Date: Sun, 01 Jul 2018 21:13:08 -0700 (PDT)
Subject: [pypy-commit] pypy unicode-utf8-py3: fixes from module/_sre/test
Message-ID: <5b39a654.1c69fb81.7fbb1.b381@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: unicode-utf8-py3
Changeset: r94802:1a502b1e961b
Date: 2018-07-01 23:11 -0500
http://bitbucket.org/pypy/pypy/changeset/1a502b1e961b/

Log:	fixes from module/_sre/test

diff --git a/pypy/module/_sre/interp_sre.py b/pypy/module/_sre/interp_sre.py
--- a/pypy/module/_sre/interp_sre.py
+++ b/pypy/module/_sre/interp_sre.py
@@ -47,6 +47,8 @@
             s = ctx._utf8[start:end]
             lgt = rutf8.get_utf8_length(s)
             return space.newutf8(s, lgt)
+        elif isinstance(ctx, rsre_core.UnicodeMatchContext):
+            return space.newtext(ctx._unicodestr[start:end])
         else:
             # unreachable
             raise SystemError
@@ -742,6 +744,8 @@
         elif isinstance(ctx, rsre_utf8.Utf8MatchContext):
             lgt = rutf8.get_utf8_length(ctx._utf8)
             return space.newutf8(ctx._utf8, lgt)
+        elif isinstance(ctx, rsre_core.UnicodeMatchContext):
+            return space.newtext(ctx._unicodestr)
         else:
             raise SystemError
 

From pypy.commits at gmail.com  Tue Jul  3 15:46:10 2018
From: pypy.commits at gmail.com (arigo)
Date: Tue, 03 Jul 2018 12:46:10 -0700 (PDT)
Subject: [pypy-commit] pypy default: Fix rvmprof/dummy: stop_sampling() is
 supposed to return an integer, not None
Message-ID: <5b3bd282.1c69fb81.adeda.ef47@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r94803:60d37209763d
Date: 2018-07-03 21:45 +0200
http://bitbucket.org/pypy/pypy/changeset/60d37209763d/

Log:	Fix rvmprof/dummy: stop_sampling() is supposed to return an integer,
	not None

diff --git a/rpython/rlib/rvmprof/dummy.py b/rpython/rlib/rvmprof/dummy.py
--- a/rpython/rlib/rvmprof/dummy.py
+++ b/rpython/rlib/rvmprof/dummy.py
@@ -23,4 +23,4 @@
         pass
 
     def stop_sampling(self):
-        pass
+        return -1

From pypy.commits at gmail.com  Thu Jul  5 10:48:47 2018
From: pypy.commits at gmail.com (arigo)
Date: Thu, 05 Jul 2018 07:48:47 -0700 (PDT)
Subject: [pypy-commit] pypy default: sys.excepthook() should add a newline
 to stdout if needed, in addition to flushing it
Message-ID: <5b3e2fcf.1c69fb81.dcafb.8f4f@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r94804:96023743535d
Date: 2018-07-05 16:47 +0200
http://bitbucket.org/pypy/pypy/changeset/96023743535d/

Log:	sys.excepthook() should add a newline to stdout if needed, in
	addition to flushing it

diff --git a/pypy/module/sys/app.py b/pypy/module/sys/app.py
--- a/pypy/module/sys/app.py
+++ b/pypy/module/sys/app.py
@@ -11,6 +11,11 @@
 
     # Flush stdout as well, both files may refer to the same file
     try:
+        if sys.stdout.softspace:
+            print
+    except:
+        pass
+    try:
         sys.stdout.flush()
     except:
         pass
diff --git a/pypy/module/sys/test/test_sysmodule.py b/pypy/module/sys/test/test_sysmodule.py
--- a/pypy/module/sys/test/test_sysmodule.py
+++ b/pypy/module/sys/test/test_sysmodule.py
@@ -256,6 +256,30 @@
             print repr(err.getvalue())
             assert err.getvalue().endswith("ValueError: %s\n" % expectedoutput)
 
+    def test_excepthook_flushes_stdout(self):
+        import sys, cStringIO
+        savestdout = sys.stdout
+        out = cStringIO.StringIO()
+        sys.stdout = out
+
+        eh = sys.__excepthook__
+
+        try:
+            raise ValueError(42)
+        except ValueError as exc:
+            print "hello"     # with end-of-line
+            eh(*sys.exc_info())
+        try:
+            raise ValueError(42)
+        except ValueError as exc:
+            print 123, 456,     # no end-of-line here
+            assert sys.stdout.softspace
+            eh(*sys.exc_info())
+            assert not sys.stdout.softspace
+
+        sys.stdout = savestdout
+        assert out.getvalue() == 'hello\n123 456\n'   # with a final \n added
+
     # FIXME: testing the code for a lost or replaced excepthook in
     # Python/pythonrun.c::PyErr_PrintEx() is tricky.
 

From pypy.commits at gmail.com  Thu Jul  5 11:02:54 2018
From: pypy.commits at gmail.com (arigo)
Date: Thu, 05 Jul 2018 08:02:54 -0700 (PDT)
Subject: [pypy-commit] pypy default: Issue #2837
Message-ID: <5b3e331e.1c69fb81.3afa9.6162@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r94805:6ee028a07651
Date: 2018-07-05 17:01 +0200
http://bitbucket.org/pypy/pypy/changeset/6ee028a07651/

Log:	Issue #2837

	Write an end-of-line and flush stdout when an exception is about to
	be printed in code.py. Needed to emulate CPython's interactive
	command line behaviour more closely, and good to avoid strange
	effects.

diff --git a/lib-python/2.7/code.py b/lib-python/2.7/code.py
--- a/lib-python/2.7/code.py
+++ b/lib-python/2.7/code.py
@@ -104,6 +104,12 @@
         except SystemExit:
             raise
         except:
+            if softspace(sys.stdout, 0):
+                print
+            try:
+                sys.stdout.flush()
+            except:
+                pass
             self.showtraceback()
         else:
             if softspace(sys.stdout, 0):
diff --git a/pypy/module/test_lib_pypy/test_code_extra.py b/pypy/module/test_lib_pypy/test_code_extra.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/test_lib_pypy/test_code_extra.py
@@ -0,0 +1,19 @@
+import py
+import sys
+import cStringIO
+import code
+
+
+def test_flush_stdout_on_error():
+    runner = code.InteractiveInterpreter()
+    old_stdout = sys.stdout
+    try:
+        mystdout = cStringIO.StringIO()
+        sys.stdout = mystdout
+        runner.runcode(compile("print 5,;0/0", "<interactive>", "exec"))
+    finally:
+        sys.stdout = old_stdout
+
+    if '__pypy__' not in sys.builtin_module_names:
+        py.test.skip('pypy only test')
+    assert mystdout.getvalue() == "5\n"

From pypy.commits at gmail.com  Thu Jul  5 11:30:51 2018
From: pypy.commits at gmail.com (arigo)
Date: Thu, 05 Jul 2018 08:30:51 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: hg merge default
Message-ID: <5b3e39ab.1c69fb81.d4c94.f16c@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r94806:87b7c6c07f22
Date: 2018-07-05 17:28 +0200
http://bitbucket.org/pypy/pypy/changeset/87b7c6c07f22/

Log:	hg merge default

	(issue #2837 not closed, needs different fix in pypy3)

diff --git a/pypy/module/sys/test/test_sysmodule.py b/pypy/module/sys/test/test_sysmodule.py
--- a/pypy/module/sys/test/test_sysmodule.py
+++ b/pypy/module/sys/test/test_sysmodule.py
@@ -358,6 +358,29 @@
             print(ascii(err.getvalue()))
             assert err.getvalue().endswith("ValueError: %s\n" % input)
 
+    def test_excepthook_flushes_stdout(self): r"""
+        import sys, io
+        savestdout = sys.stdout
+        out = io.StringIO()
+        sys.stdout = out
+
+        eh = sys.__excepthook__
+
+        try:
+            raise ValueError(42)
+        except ValueError as exc:
+            print("hello")     # with end-of-line
+            eh(*sys.exc_info())
+        try:
+            raise ValueError(42)
+        except ValueError as exc:
+            print(123, 456, end="")     # no end-of-line here
+            eh(*sys.exc_info())
+
+        sys.stdout = savestdout
+        assert out.getvalue() == 'hello\n123 456'   # no final \n added in 3.x
+        """
+
     # FIXME: testing the code for a lost or replaced excepthook in
     # Python/pythonrun.c::PyErr_PrintEx() is tricky.
 
diff --git a/rpython/rlib/rvmprof/dummy.py b/rpython/rlib/rvmprof/dummy.py
--- a/rpython/rlib/rvmprof/dummy.py
+++ b/rpython/rlib/rvmprof/dummy.py
@@ -23,4 +23,4 @@
         pass
 
     def stop_sampling(self):
-        pass
+        return -1

From pypy.commits at gmail.com  Thu Jul  5 11:30:53 2018
From: pypy.commits at gmail.com (arigo)
Date: Thu, 05 Jul 2018 08:30:53 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Fix for issue #2837 in pypy3
Message-ID: <5b3e39ad.1c69fb81.bb4d9.a9cc@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r94807:a74f33cd9f8f
Date: 2018-07-05 17:29 +0200
http://bitbucket.org/pypy/pypy/changeset/a74f33cd9f8f/

Log:	Fix for issue #2837 in pypy3

diff --git a/lib_pypy/pyrepl/simple_interact.py b/lib_pypy/pyrepl/simple_interact.py
--- a/lib_pypy/pyrepl/simple_interact.py
+++ b/lib_pypy/pyrepl/simple_interact.py
@@ -81,3 +81,8 @@
         except MemoryError:
             console.write("\nMemoryError\n")
             console.resetbuffer()
+        finally:
+            try:
+                sys.stdout.flush()
+            except:
+                pass

From pypy.commits at gmail.com  Thu Jul  5 11:34:53 2018
From: pypy.commits at gmail.com (arigo)
Date: Thu, 05 Jul 2018 08:34:53 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Flush stdout also before the first prompt,
 because why not. The
Message-ID: <5b3e3a9d.1c69fb81.a6569.9283@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r94808:180dc8d9dd61
Date: 2018-07-05 17:34 +0200
http://bitbucket.org/pypy/pypy/changeset/180dc8d9dd61/

Log:	Flush stdout also before the first prompt, because why not. The
	difference is visible by running:

	 python3 -i -c "print(5,end='')"

	which is a very minor bug in CPython (3.5.3)

diff --git a/lib_pypy/pyrepl/simple_interact.py b/lib_pypy/pyrepl/simple_interact.py
--- a/lib_pypy/pyrepl/simple_interact.py
+++ b/lib_pypy/pyrepl/simple_interact.py
@@ -66,6 +66,10 @@
 
     while 1:
         try:
+            try:
+                sys.stdout.flush()
+            except:
+                pass
             ps1 = getattr(sys, 'ps1', '>>> ')
             ps2 = getattr(sys, 'ps2', '... ')
             try:
@@ -81,8 +85,3 @@
         except MemoryError:
             console.write("\nMemoryError\n")
             console.resetbuffer()
-        finally:
-            try:
-                sys.stdout.flush()
-            except:
-                pass

From pypy.commits at gmail.com  Thu Jul  5 21:30:58 2018
From: pypy.commits at gmail.com (wlav)
Date: Thu, 05 Jul 2018 18:30:58 -0700 (PDT)
Subject: [pypy-commit] pypy cppyy-packaging: merge default into
 cppyy-packaging
Message-ID: <5b3ec652.1c69fb81.fc27f.f301@mx.google.com>

Author: Wim Lavrijsen <WLavrijsen at lbl.gov>
Branch: cppyy-packaging
Changeset: r94810:935c38f271a8
Date: 2018-07-05 17:37 -0700
http://bitbucket.org/pypy/pypy/changeset/935c38f271a8/

Log:	merge default into cppyy-packaging

diff --git a/lib-python/2.7/code.py b/lib-python/2.7/code.py
--- a/lib-python/2.7/code.py
+++ b/lib-python/2.7/code.py
@@ -104,6 +104,12 @@
         except SystemExit:
             raise
         except:
+            if softspace(sys.stdout, 0):
+                print
+            try:
+                sys.stdout.flush()
+            except:
+                pass
             self.showtraceback()
         else:
             if softspace(sys.stdout, 0):
diff --git a/pypy/doc/sandbox.rst b/pypy/doc/sandbox.rst
--- a/pypy/doc/sandbox.rst
+++ b/pypy/doc/sandbox.rst
@@ -3,6 +3,11 @@
 PyPy's sandboxing features
 ==========================
 
+.. warning:: This is not actively maintained. You will likely have to fix
+   some issues yourself, or otherwise play around on your own. We provide
+   this documentation for historical reasions, it will not translate or
+   run on the latest PyPy code base.
+
 Introduction
 ------------
 
diff --git a/pypy/interpreter/test/test_raise.py b/pypy/interpreter/test/test_raise.py
--- a/pypy/interpreter/test/test_raise.py
+++ b/pypy/interpreter/test/test_raise.py
@@ -280,3 +280,15 @@
             def __new__(cls, *args):
                 return object()
         raises(TypeError, "raise MyException")
+
+    def test_with_exit_True(self):
+        class X:
+            def __enter__(self):
+                pass
+            def __exit__(self, *args):
+                return True
+        def g():
+            with X():
+                return 42
+            assert False, "unreachable"
+        assert g() == 42
diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -643,7 +643,7 @@
     '_PyTraceMalloc_Track', '_PyTraceMalloc_Untrack', 'PyMem_Malloc',
     'PyObject_Free', 'PyObject_GC_Del', 'PyType_GenericAlloc',
     '_PyObject_New', '_PyObject_NewVar',
-    '_PyObject_GC_New', '_PyObject_GC_NewVar',
+    '_PyObject_GC_Malloc', '_PyObject_GC_New', '_PyObject_GC_NewVar',
     'PyObject_Init', 'PyObject_InitVar', 'PyInt_FromLong',
     'PyTuple_New', '_Py_Dealloc',
 ]
@@ -772,6 +772,9 @@
 # a pointer to PyObject
 PyObjectP = rffi.CArrayPtr(PyObject)
 
+# int *
+INTP_real = rffi.CArrayPtr(rffi.INT_real)
+
 def configure_types():
     for config in (CConfig, CConfig2):
         for name, TYPE in rffi_platform.configure(config).iteritems():
diff --git a/pypy/module/cpyext/include/object.h b/pypy/module/cpyext/include/object.h
--- a/pypy/module/cpyext/include/object.h
+++ b/pypy/module/cpyext/include/object.h
@@ -383,6 +383,7 @@
 
 PyAPI_FUNC(PyObject *) _PyObject_New(PyTypeObject *);
 PyAPI_FUNC(PyVarObject *) _PyObject_NewVar(PyTypeObject *, Py_ssize_t);
+PyAPI_FUNC(PyObject *) _PyObject_GC_Malloc(size_t);
 PyAPI_FUNC(PyObject *) _PyObject_GC_New(PyTypeObject *);
 PyAPI_FUNC(PyVarObject *) _PyObject_GC_NewVar(PyTypeObject *, Py_ssize_t);
 
diff --git a/pypy/module/cpyext/longobject.py b/pypy/module/cpyext/longobject.py
--- a/pypy/module/cpyext/longobject.py
+++ b/pypy/module/cpyext/longobject.py
@@ -1,7 +1,7 @@
 from rpython.rtyper.lltypesystem import lltype, rffi
 from pypy.module.cpyext.api import (
     cpython_api, PyObject, build_type_checkers_flags, Py_ssize_t,
-    CONST_STRING, ADDR, CANNOT_FAIL)
+    CONST_STRING, ADDR, CANNOT_FAIL, INTP_real)
 from pypy.objspace.std.longobject import W_LongObject
 from pypy.interpreter.error import OperationError, oefmt
 from pypy.module.cpyext.intobject import PyInt_AsUnsignedLongMask
@@ -112,7 +112,7 @@
     num = space.bigint_w(w_long)
     return num.ulonglongmask()
 
- at cpython_api([PyObject, rffi.CArrayPtr(rffi.INT_real)], lltype.Signed,
+ at cpython_api([PyObject, INTP_real], lltype.Signed,
              error=-1)
 def PyLong_AsLongAndOverflow(space, w_long, overflow_ptr):
     """
@@ -133,7 +133,7 @@
         overflow_ptr[0] = rffi.cast(rffi.INT_real, -1)
     return -1
 
- at cpython_api([PyObject, rffi.CArrayPtr(rffi.INT_real)], rffi.LONGLONG,
+ at cpython_api([PyObject, INTP_real], rffi.LONGLONG,
              error=-1)
 def PyLong_AsLongLongAndOverflow(space, w_long, overflow_ptr):
     """
diff --git a/pypy/module/cpyext/object.py b/pypy/module/cpyext/object.py
--- a/pypy/module/cpyext/object.py
+++ b/pypy/module/cpyext/object.py
@@ -1,7 +1,7 @@
 from rpython.rtyper.lltypesystem import rffi, lltype
 from pypy.module.cpyext.api import (
     cpython_api, generic_cpy_call, CANNOT_FAIL, Py_ssize_t, Py_ssize_tP,
-    PyVarObject, size_t, slot_function,
+    PyVarObject, size_t, slot_function, INTP_real,
     Py_TPFLAGS_HEAPTYPE, Py_LT, Py_LE, Py_EQ, Py_NE, Py_GT,
     Py_GE, CONST_STRING, FILEP, fwrite, c_only)
 from pypy.module.cpyext.pyobject import (
@@ -221,14 +221,14 @@
     expression cmp(o1, o2)."""
     return space.int_w(space.cmp(w_o1, w_o2))
 
- at cpython_api([PyObject, PyObject, rffi.INTP], rffi.INT_real, error=-1)
+ at cpython_api([PyObject, PyObject, INTP_real], rffi.INT_real, error=-1)
 def PyObject_Cmp(space, w_o1, w_o2, result):
     """Compare the values of o1 and o2 using a routine provided by o1, if one
     exists, otherwise with a routine provided by o2.  The result of the
     comparison is returned in result.  Returns -1 on failure.  This is the
     equivalent of the Python statement result = cmp(o1, o2)."""
     res = space.int_w(space.cmp(w_o1, w_o2))
-    result[0] = rffi.cast(rffi.INT, res)
+    result[0] = rffi.cast(rffi.INT_real, res)
     return 0
 
 @cpython_api([PyObject, PyObject, rffi.INT_real], PyObject)
diff --git a/pypy/module/cpyext/pystrtod.py b/pypy/module/cpyext/pystrtod.py
--- a/pypy/module/cpyext/pystrtod.py
+++ b/pypy/module/cpyext/pystrtod.py
@@ -1,6 +1,6 @@
 import errno
 from pypy.interpreter.error import oefmt
-from pypy.module.cpyext.api import cpython_api, CONST_STRING
+from pypy.module.cpyext.api import cpython_api, CONST_STRING, INTP_real
 from pypy.module.cpyext.pyobject import PyObject
 from rpython.rlib import rdtoa
 from rpython.rlib import rfloat
@@ -80,7 +80,7 @@
         if not user_endptr:
             lltype.free(endptr, flavor='raw')
 
- at cpython_api([rffi.DOUBLE, lltype.Char, rffi.INT_real, rffi.INT_real, rffi.INTP], rffi.CCHARP)
+ at cpython_api([rffi.DOUBLE, lltype.Char, rffi.INT_real, rffi.INT_real, INTP_real], rffi.CCHARP)
 def PyOS_double_to_string(space, val, format_code, precision, flags, ptype):
     """Convert a double val to a string using supplied
     format_code, precision, and flags.
@@ -114,7 +114,7 @@
     buffer, rtype = rfloat.double_to_string(val, format_code,
                                             intmask(precision),
                                             intmask(flags))
-    if ptype != lltype.nullptr(rffi.INTP.TO):
-        ptype[0] = rffi.cast(rffi.INT, DOUBLE_TO_STRING_TYPES_MAP[rtype])
+    if ptype != lltype.nullptr(INTP_real.TO):
+        ptype[0] = rffi.cast(rffi.INT_real, DOUBLE_TO_STRING_TYPES_MAP[rtype])
     bufp = rffi.str2charp(buffer)
     return bufp
diff --git a/pypy/module/cpyext/src/object.c b/pypy/module/cpyext/src/object.c
--- a/pypy/module/cpyext/src/object.c
+++ b/pypy/module/cpyext/src/object.c
@@ -60,6 +60,11 @@
     return (PyObject*)_PyObject_NewVar(type, 0);
 }
 
+PyObject * _PyObject_GC_Malloc(size_t size)
+{
+    return (PyObject *)PyObject_Malloc(size);
+}
+
 PyObject * _PyObject_GC_New(PyTypeObject *type)
 {
     return _PyObject_New(type);
diff --git a/pypy/module/cpyext/test/test_object.py b/pypy/module/cpyext/test/test_object.py
--- a/pypy/module/cpyext/test/test_object.py
+++ b/pypy/module/cpyext/test/test_object.py
@@ -5,7 +5,7 @@
 from rpython.rtyper.lltypesystem import rffi, lltype
 from pypy.module.cpyext.pyobject import get_w_obj_and_decref
 from pypy.module.cpyext.api import (
-    Py_LT, Py_LE, Py_NE, Py_EQ, Py_GE, Py_GT)
+    Py_LT, Py_LE, Py_NE, Py_EQ, Py_GE, Py_GT, INTP_real)
 from pypy.module.cpyext.object import (
     PyObject_IsTrue, PyObject_Not, PyObject_GetAttrString,
     PyObject_DelAttrString, PyObject_GetAttr, PyObject_DelAttr,
@@ -205,7 +205,7 @@
 
     def test_cmp(self, space, api):
         w = space.wrap
-        with lltype.scoped_alloc(rffi.INTP.TO, 1) as ptr:
+        with lltype.scoped_alloc(INTP_real.TO, 1) as ptr:
             assert api.PyObject_Cmp(w(42), w(72), ptr) == 0
             assert ptr[0] == -1
             assert api.PyObject_Cmp(w("a"), w("a"), ptr) == 0
diff --git a/pypy/module/cpyext/test/test_pystrtod.py b/pypy/module/cpyext/test/test_pystrtod.py
--- a/pypy/module/cpyext/test/test_pystrtod.py
+++ b/pypy/module/cpyext/test/test_pystrtod.py
@@ -4,7 +4,7 @@
 from pypy.module.cpyext.test.test_api import BaseApiTest, raises_w
 from rpython.rtyper.lltypesystem import rffi
 from rpython.rtyper.lltypesystem import lltype
-from pypy.module.cpyext.pystrtod import PyOS_string_to_double
+from pypy.module.cpyext.pystrtod import PyOS_string_to_double, INTP_real
 
 
 class TestPyOS_string_to_double(BaseApiTest):
@@ -90,7 +90,7 @@
 class TestPyOS_double_to_string(BaseApiTest):
 
     def test_format_code(self, api):
-        ptype = lltype.malloc(rffi.INTP.TO, 1, flavor='raw')
+        ptype = lltype.malloc(INTP_real.TO, 1, flavor='raw')
         r = api.PyOS_double_to_string(150.0, 'e', 1, 0, ptype)
         assert '1.5e+02' == rffi.charp2str(r)
         type_value = rffi.cast(lltype.Signed, ptype[0])
@@ -99,7 +99,7 @@
         lltype.free(ptype, flavor='raw')
 
     def test_precision(self, api):
-        ptype = lltype.malloc(rffi.INTP.TO, 1, flavor='raw')
+        ptype = lltype.malloc(INTP_real.TO, 1, flavor='raw')
         r = api.PyOS_double_to_string(3.14159269397, 'g', 5, 0, ptype)
         assert '3.1416' == rffi.charp2str(r)
         type_value = rffi.cast(lltype.Signed, ptype[0])
@@ -108,7 +108,7 @@
         lltype.free(ptype, flavor='raw')
 
     def test_flags_sign(self, api):
-        ptype = lltype.malloc(rffi.INTP.TO, 1, flavor='raw')
+        ptype = lltype.malloc(INTP_real.TO, 1, flavor='raw')
         r = api.PyOS_double_to_string(-3.14, 'g', 3, 1, ptype)
         assert '-3.14' == rffi.charp2str(r)
         type_value = rffi.cast(lltype.Signed, ptype[0])
@@ -117,7 +117,7 @@
         lltype.free(ptype, flavor='raw')
 
     def test_flags_add_dot_0(self, api):
-        ptype = lltype.malloc(rffi.INTP.TO, 1, flavor='raw')
+        ptype = lltype.malloc(INTP_real.TO, 1, flavor='raw')
         r = api.PyOS_double_to_string(3, 'g', 5, 2, ptype)
         assert '3.0' == rffi.charp2str(r)
         type_value = rffi.cast(lltype.Signed, ptype[0])
@@ -126,7 +126,7 @@
         lltype.free(ptype, flavor='raw')
 
     def test_flags_alt(self, api):
-        ptype = lltype.malloc(rffi.INTP.TO, 1, flavor='raw')
+        ptype = lltype.malloc(INTP_real.TO, 1, flavor='raw')
         r = api.PyOS_double_to_string(314., 'g', 3, 4, ptype)
         assert '314.' == rffi.charp2str(r)
         type_value = rffi.cast(lltype.Signed, ptype[0])
@@ -135,7 +135,7 @@
         lltype.free(ptype, flavor='raw')
 
     def test_ptype_nan(self, api):
-        ptype = lltype.malloc(rffi.INTP.TO, 1, flavor='raw')
+        ptype = lltype.malloc(INTP_real.TO, 1, flavor='raw')
         r = api.PyOS_double_to_string(float('nan'), 'g', 3, 4, ptype)
         assert 'nan' == rffi.charp2str(r)
         type_value = rffi.cast(lltype.Signed, ptype[0])
@@ -144,7 +144,7 @@
         lltype.free(ptype, flavor='raw')
 
     def test_ptype_infinity(self, api):
-        ptype = lltype.malloc(rffi.INTP.TO, 1, flavor='raw')
+        ptype = lltype.malloc(INTP_real.TO, 1, flavor='raw')
         r = api.PyOS_double_to_string(1e200 * 1e200, 'g', 0, 0, ptype)
         assert 'inf' == rffi.charp2str(r)
         type_value = rffi.cast(lltype.Signed, ptype[0])
@@ -153,8 +153,8 @@
         lltype.free(ptype, flavor='raw')
 
     def test_ptype_null(self, api):
-        ptype = lltype.nullptr(rffi.INTP.TO)
+        ptype = lltype.nullptr(INTP_real.TO)
         r = api.PyOS_double_to_string(3.14, 'g', 3, 0, ptype)
         assert '3.14' == rffi.charp2str(r)
-        assert ptype == lltype.nullptr(rffi.INTP.TO)
+        assert ptype == lltype.nullptr(INTP_real.TO)
         rffi.free_charp(r)
diff --git a/pypy/module/cpyext/test/test_unicodeobject.py b/pypy/module/cpyext/test/test_unicodeobject.py
--- a/pypy/module/cpyext/test/test_unicodeobject.py
+++ b/pypy/module/cpyext/test/test_unicodeobject.py
@@ -4,7 +4,7 @@
 from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
 from pypy.module.cpyext.unicodeobject import (
     Py_UNICODE, PyUnicodeObject, new_empty_unicode)
-from pypy.module.cpyext.api import PyObjectP, PyObject
+from pypy.module.cpyext.api import PyObjectP, PyObject, INTP_real
 from pypy.module.cpyext.pyobject import decref, from_ref
 from rpython.rtyper.lltypesystem import rffi, lltype
 import sys, py
@@ -464,8 +464,8 @@
                     value = 1
                 else:
                     value = 0
-                pendian = lltype.malloc(rffi.INTP.TO, 1, flavor='raw')
-                pendian[0] = rffi.cast(rffi.INT, value)
+                pendian = lltype.malloc(INTP_real.TO, 1, flavor='raw')
+                pendian[0] = rffi.cast(rffi.INT_real, value)
             else:
                 pendian = None
 
@@ -477,7 +477,7 @@
             rffi.free_charp(strict_charp)
             if pendian:
                 if realendian is not None:
-                    assert rffi.cast(rffi.INT, realendian) == pendian[0]
+                    assert rffi.cast(rffi.INT_real, realendian) == pendian[0]
                 lltype.free(pendian, flavor='raw')
 
         test("\x61\x00\x62\x00\x63\x00\x64\x00", -1)
@@ -500,8 +500,8 @@
                     value = 1
                 else:
                     value = 0
-                pendian = lltype.malloc(rffi.INTP.TO, 1, flavor='raw')
-                pendian[0] = rffi.cast(rffi.INT, value)
+                pendian = lltype.malloc(INTP_real.TO, 1, flavor='raw')
+                pendian[0] = rffi.cast(rffi.INT_real, value)
             else:
                 pendian = None
 
diff --git a/pypy/module/cpyext/unicodeobject.py b/pypy/module/cpyext/unicodeobject.py
--- a/pypy/module/cpyext/unicodeobject.py
+++ b/pypy/module/cpyext/unicodeobject.py
@@ -3,7 +3,7 @@
 from pypy.module.unicodedata import unicodedb
 from pypy.module.cpyext.api import (
     CANNOT_FAIL, Py_ssize_t, build_type_checkers_flags, cpython_api,
-    bootstrap_function, CONST_STRING,
+    bootstrap_function, CONST_STRING, INTP_real,
     CONST_WSTRING, slot_function, cts, parse_dir)
 from pypy.module.cpyext.pyerrors import PyErr_BadArgument
 from pypy.module.cpyext.pyobject import (
@@ -526,7 +526,7 @@
 if sys.platform == 'win32':
     make_conversion_functions('MBCS', 'mbcs')
 
- at cpython_api([CONST_STRING, Py_ssize_t, CONST_STRING, rffi.INTP], PyObject)
+ at cpython_api([CONST_STRING, Py_ssize_t, CONST_STRING, INTP_real], PyObject)
 def PyUnicode_DecodeUTF16(space, s, size, llerrors, pbyteorder):
     """Decode length bytes from a UTF-16 encoded buffer string and return the
     corresponding Unicode object.  errors (if non-NULL) defines the error
@@ -576,11 +576,11 @@
         None, # errorhandler
         byteorder)
     if pbyteorder is not None:
-        pbyteorder[0] = rffi.cast(rffi.INT, byteorder)
+        pbyteorder[0] = rffi.cast(rffi.INT_real, byteorder)
 
     return space.newunicode(result)
 
- at cpython_api([CONST_STRING, Py_ssize_t, CONST_STRING, rffi.INTP], PyObject)
+ at cpython_api([CONST_STRING, Py_ssize_t, CONST_STRING, INTP_real], PyObject)
 def PyUnicode_DecodeUTF32(space, s, size, llerrors, pbyteorder):
     """Decode length bytes from a UTF-32 encoded buffer string and
     return the corresponding Unicode object.  errors (if non-NULL)
@@ -632,7 +632,7 @@
         None, # errorhandler
         byteorder)
     if pbyteorder is not None:
-        pbyteorder[0] = rffi.cast(rffi.INT, byteorder)
+        pbyteorder[0] = rffi.cast(rffi.INT_real, byteorder)
 
     return space.newunicode(result)
 
diff --git a/pypy/module/sys/app.py b/pypy/module/sys/app.py
--- a/pypy/module/sys/app.py
+++ b/pypy/module/sys/app.py
@@ -11,6 +11,11 @@
 
     # Flush stdout as well, both files may refer to the same file
     try:
+        if sys.stdout.softspace:
+            print
+    except:
+        pass
+    try:
         sys.stdout.flush()
     except:
         pass
diff --git a/pypy/module/sys/test/test_sysmodule.py b/pypy/module/sys/test/test_sysmodule.py
--- a/pypy/module/sys/test/test_sysmodule.py
+++ b/pypy/module/sys/test/test_sysmodule.py
@@ -256,6 +256,30 @@
             print repr(err.getvalue())
             assert err.getvalue().endswith("ValueError: %s\n" % expectedoutput)
 
+    def test_excepthook_flushes_stdout(self):
+        import sys, cStringIO
+        savestdout = sys.stdout
+        out = cStringIO.StringIO()
+        sys.stdout = out
+
+        eh = sys.__excepthook__
+
+        try:
+            raise ValueError(42)
+        except ValueError as exc:
+            print "hello"     # with end-of-line
+            eh(*sys.exc_info())
+        try:
+            raise ValueError(42)
+        except ValueError as exc:
+            print 123, 456,     # no end-of-line here
+            assert sys.stdout.softspace
+            eh(*sys.exc_info())
+            assert not sys.stdout.softspace
+
+        sys.stdout = savestdout
+        assert out.getvalue() == 'hello\n123 456\n'   # with a final \n added
+
     # FIXME: testing the code for a lost or replaced excepthook in
     # Python/pythonrun.c::PyErr_PrintEx() is tricky.
 
diff --git a/pypy/module/test_lib_pypy/test_code_extra.py b/pypy/module/test_lib_pypy/test_code_extra.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/test_lib_pypy/test_code_extra.py
@@ -0,0 +1,19 @@
+import py
+import sys
+import cStringIO
+import code
+
+
+def test_flush_stdout_on_error():
+    runner = code.InteractiveInterpreter()
+    old_stdout = sys.stdout
+    try:
+        mystdout = cStringIO.StringIO()
+        sys.stdout = mystdout
+        runner.runcode(compile("print 5,;0/0", "<interactive>", "exec"))
+    finally:
+        sys.stdout = old_stdout
+
+    if '__pypy__' not in sys.builtin_module_names:
+        py.test.skip('pypy only test')
+    assert mystdout.getvalue() == "5\n"
diff --git a/rpython/rlib/rvmprof/dummy.py b/rpython/rlib/rvmprof/dummy.py
--- a/rpython/rlib/rvmprof/dummy.py
+++ b/rpython/rlib/rvmprof/dummy.py
@@ -23,4 +23,4 @@
         pass
 
     def stop_sampling(self):
-        pass
+        return -1

From pypy.commits at gmail.com  Thu Jul  5 21:31:01 2018
From: pypy.commits at gmail.com (wlav)
Date: Thu, 05 Jul 2018 18:31:01 -0700 (PDT)
Subject: [pypy-commit] pypy default: merge cppyy-packaging: fixes
 performance regression (see:
 https://bitbucket.org/wlav/cppyy/issues/34/class-member-function-calls-not-being)
Message-ID: <5b3ec655.1c69fb81.f3bc1.140f@mx.google.com>

Author: Wim Lavrijsen <WLavrijsen at lbl.gov>
Branch: 
Changeset: r94811:53d95e0b1421
Date: 2018-07-05 18:11 -0700
http://bitbucket.org/pypy/pypy/changeset/53d95e0b1421/

Log:	merge cppyy-packaging: fixes performance regression (see:
	https://bitbucket.org/wlav/cppyy/issues/34/class-member-function-
	calls-not-being)

diff --git a/pypy/module/_cppyy/converter.py b/pypy/module/_cppyy/converter.py
--- a/pypy/module/_cppyy/converter.py
+++ b/pypy/module/_cppyy/converter.py
@@ -82,10 +82,9 @@
 
 
 class TypeConverter(object):
-    _immutable_fields_ = ['cffi_name', 'uses_local', 'name']
+    _immutable_fields_ = ['cffi_name', 'name']
 
     cffi_name  = None
-    uses_local = False
     name       = ""
 
     def __init__(self, space, extra):
@@ -108,10 +107,10 @@
         from pypy.module._cppyy.interp_cppyy import FastCallNotPossible
         raise FastCallNotPossible
 
-    def convert_argument(self, space, w_obj, address, call_local):
+    def convert_argument(self, space, w_obj, address):
         self._is_abstract(space)
 
-    def convert_argument_libffi(self, space, w_obj, address, call_local):
+    def convert_argument_libffi(self, space, w_obj, address, scratch):
         from pypy.module._cppyy.interp_cppyy import FastCallNotPossible
         raise FastCallNotPossible
 
@@ -125,10 +124,10 @@
     def to_memory(self, space, w_obj, w_value, offset):
         self._is_abstract(space)
 
-    def finalize_call(self, space, w_obj, call_local):
+    def finalize_call(self, space, w_obj):
         pass
 
-    def free_argument(self, space, arg, call_local):
+    def free_argument(self, space, arg):
         pass
 
 
@@ -172,7 +171,7 @@
         state = space.fromcache(ffitypes.State)
         return state.c_voidp
 
-    def convert_argument(self, space, w_obj, address, call_local):
+    def convert_argument(self, space, w_obj, address):
         w_tc = space.findattr(w_obj, space.newtext('typecode'))
         if w_tc is not None and space.text_w(w_tc) != self.typecode:
             raise oefmt(space.w_TypeError,
@@ -208,7 +207,7 @@
 class NumericTypeConverterMixin(object):
     _mixin_ = True
 
-    def convert_argument_libffi(self, space, w_obj, address, call_local):
+    def convert_argument_libffi(self, space, w_obj, address, scratch):
         x = rffi.cast(self.c_ptrtype, address)
         x[0] = self._unwrap_object(space, w_obj)
 
@@ -228,26 +227,23 @@
 
 class ConstRefNumericTypeConverterMixin(NumericTypeConverterMixin):
     _mixin_ = True
-    _immutable_fields_ = ['uses_local']
-
-    uses_local = True
 
     def cffi_type(self, space):
         state = space.fromcache(ffitypes.State)
         return state.c_voidp
 
-    def convert_argument_libffi(self, space, w_obj, address, call_local):
-        assert rffi.sizeof(self.c_type) <= 2*rffi.sizeof(rffi.VOIDP)  # see interp_cppyy.py
+    def convert_argument_libffi(self, space, w_obj, address, scratch):
         obj = self._unwrap_object(space, w_obj)
-        typed_buf = rffi.cast(self.c_ptrtype, call_local)
+        typed_buf = rffi.cast(self.c_ptrtype, scratch)
         typed_buf[0] = obj
         x = rffi.cast(rffi.VOIDPP, address)
-        x[0] = call_local
+        x[0] = scratch
+
 
 class IntTypeConverterMixin(NumericTypeConverterMixin):
     _mixin_ = True
 
-    def convert_argument(self, space, w_obj, address, call_local):
+    def convert_argument(self, space, w_obj, address):
         x = rffi.cast(self.c_ptrtype, address)
         x[0] = self._unwrap_object(space, w_obj)
         ba = rffi.cast(rffi.CCHARP, address)
@@ -256,7 +252,7 @@
 class FloatTypeConverterMixin(NumericTypeConverterMixin):
     _mixin_ = True
 
-    def convert_argument(self, space, w_obj, address, call_local):
+    def convert_argument(self, space, w_obj, address):
         x = rffi.cast(self.c_ptrtype, address)
         x[0] = self._unwrap_object(space, w_obj)
         ba = rffi.cast(rffi.CCHARP, address)
@@ -273,18 +269,18 @@
         state = space.fromcache(ffitypes.State)
         return state.c_void
 
-    def convert_argument(self, space, w_obj, address, call_local):
+    def convert_argument(self, space, w_obj, address):
         self._is_abstract(space)
 
 
 class BoolConverter(ffitypes.typeid(bool), TypeConverter):
-    def convert_argument(self, space, w_obj, address, call_local):
+    def convert_argument(self, space, w_obj, address):
         x = rffi.cast(rffi.LONGP, address)
         x[0] = self._unwrap_object(space, w_obj)
         ba = rffi.cast(rffi.CCHARP, address)
         ba[capi.c_function_arg_typeoffset(space)] = 'b'
 
-    def convert_argument_libffi(self, space, w_obj, address, call_local):
+    def convert_argument_libffi(self, space, w_obj, address, scratch):
         x = rffi.cast(rffi.LONGP, address)
         x[0] = self._unwrap_object(space, w_obj)
 
@@ -303,13 +299,13 @@
             address[0] = '\x00'
 
 class CharConverter(ffitypes.typeid(rffi.CHAR), TypeConverter):
-    def convert_argument(self, space, w_obj, address, call_local):
+    def convert_argument(self, space, w_obj, address):
         x = rffi.cast(rffi.CCHARP, address)
         x[0] = self._unwrap_object(space, w_obj)
         ba = rffi.cast(rffi.CCHARP, address)
         ba[capi.c_function_arg_typeoffset(space)] = 'b'
 
-    def convert_argument_libffi(self, space, w_obj, address, call_local):
+    def convert_argument_libffi(self, space, w_obj, address, scratch):
         x = rffi.cast(self.c_ptrtype, address)
         x[0] = self._unwrap_object(space, w_obj)
 
@@ -348,7 +344,7 @@
         state = space.fromcache(ffitypes.State)
         return state.c_voidp
 
-    def convert_argument_libffi(self, space, w_obj, address, call_local):
+    def convert_argument_libffi(self, space, w_obj, address, scratch):
         from pypy.module._cppyy.interp_cppyy import FastCallNotPossible
         raise FastCallNotPossible
 
@@ -381,7 +377,7 @@
 
 
 class CStringConverter(TypeConverter):
-    def convert_argument(self, space, w_obj, address, call_local):
+    def convert_argument(self, space, w_obj, address):
         x = rffi.cast(rffi.LONGP, address)
         arg = space.text_w(w_obj)
         x[0] = rffi.cast(rffi.LONG, rffi.str2charp(arg))
@@ -393,7 +389,7 @@
         charpptr = rffi.cast(rffi.CCHARPP, address)
         return space.newtext(rffi.charp2str(charpptr[0]))
 
-    def free_argument(self, space, arg, call_local):
+    def free_argument(self, space, arg):
         lltype.free(rffi.cast(rffi.CCHARPP, arg)[0], flavor='raw')
 
 class CStringConverterWithSize(CStringConverter):
@@ -423,13 +419,13 @@
         state = space.fromcache(ffitypes.State)
         return state.c_voidp
 
-    def convert_argument(self, space, w_obj, address, call_local):
+    def convert_argument(self, space, w_obj, address):
         x = rffi.cast(rffi.VOIDPP, address)
         x[0] = self._unwrap_object(space, w_obj)
         ba = rffi.cast(rffi.CCHARP, address)
         ba[capi.c_function_arg_typeoffset(space)] = 'o'
 
-    def convert_argument_libffi(self, space, w_obj, address, call_local):
+    def convert_argument_libffi(self, space, w_obj, address, scratch):
         x = rffi.cast(rffi.VOIDPP, address)
         x[0] = self._unwrap_object(space, w_obj)
 
@@ -452,37 +448,39 @@
             address[0] = rffi.cast(rffi.VOIDP, self._unwrap_object(space, w_value))
 
 class VoidPtrPtrConverter(TypeConverter):
-    _immutable_fields_ = ['uses_local', 'typecode']
+    typecode = 'p'
 
-    uses_local = True
-    typecode = 'a'
+    def __init__(self, space, extra):
+        self.ref_buffer = lltype.nullptr(rffi.VOIDPP.TO)
 
-    def convert_argument(self, space, w_obj, address, call_local):
+    def convert_argument(self, space, w_obj, address):
         x = rffi.cast(rffi.VOIDPP, address)
-        ba = rffi.cast(rffi.CCHARP, address)
         try:
             x[0] = get_rawbuffer(space, w_obj)
         except TypeError:
-            r = rffi.cast(rffi.VOIDPP, call_local)
-            r[0] = rffi.cast(rffi.VOIDP, get_rawobject(space, w_obj))
-            x[0] = rffi.cast(rffi.VOIDP, call_local)
+            ptr = rffi.cast(rffi.VOIDP, get_rawobject(space, w_obj))
+            self.ref_buffer = lltype.malloc(rffi.VOIDPP.TO, 1, flavor='raw')
+            self.ref_buffer[0] = ptr
+            x[0] = self.ref_buffer
+        ba = rffi.cast(rffi.CCHARP, address)
         ba[capi.c_function_arg_typeoffset(space)] = self.typecode
 
-    def finalize_call(self, space, w_obj, call_local):
-        r = rffi.cast(rffi.VOIDPP, call_local)
-        try:
-            set_rawobject(space, w_obj, r[0])
-        except OperationError:
-            pass             # no set on buffer/array/None
+    def finalize_call(self, space, w_obj):
+        if self.ref_buffer:
+            set_rawobject(space, w_obj, self.ref_buffer[0])
+
+    def free_argument(self, space, arg):
+        if self.ref_buffer:
+            lltype.free(self.ref_buffer, flavor='raw')
+            self.ref_buffer = lltype.nullptr(rffi.VOIDPP.TO)
 
 class VoidPtrRefConverter(VoidPtrPtrConverter):
-    _immutable_fields_ = ['uses_local', 'typecode']
-    uses_local = True
+    _immutable_fields_ = ['typecode']
     typecode   = 'V'
 
 class InstanceRefConverter(TypeConverter):
     _immutable_fields_ = ['typecode', 'clsdecl']
-    typecode    = 'V'
+    typecode = 'V'
 
     def __init__(self, space, clsdecl):
         from pypy.module._cppyy.interp_cppyy import W_CPPClassDecl
@@ -508,14 +506,14 @@
         state = space.fromcache(ffitypes.State)
         return state.c_voidp
 
-    def convert_argument(self, space, w_obj, address, call_local):
+    def convert_argument(self, space, w_obj, address):
         x = rffi.cast(rffi.VOIDPP, address)
         x[0] = rffi.cast(rffi.VOIDP, self._unwrap_object(space, w_obj))
         address = rffi.cast(capi.C_OBJECT, address)
         ba = rffi.cast(rffi.CCHARP, address)
         ba[capi.c_function_arg_typeoffset(space)] = self.typecode
 
-    def convert_argument_libffi(self, space, w_obj, address, call_local):
+    def convert_argument_libffi(self, space, w_obj, address, scratch):
         x = rffi.cast(rffi.VOIDPP, address)
         x[0] = rffi.cast(rffi.VOIDP, self._unwrap_object(space, w_obj))
 
@@ -539,7 +537,7 @@
 
 class InstanceConverter(InstanceRefConverter):
 
-    def convert_argument_libffi(self, space, w_obj, address, call_local):
+    def convert_argument_libffi(self, space, w_obj, address, scratch):
         from pypy.module._cppyy.interp_cppyy import FastCallNotPossible
         raise FastCallNotPossible       # TODO: by-value is a jit_libffi special case
 
@@ -551,9 +549,8 @@
     def to_memory(self, space, w_obj, w_value, offset):
         self._is_abstract(space)
 
-
 class InstancePtrConverter(InstanceRefConverter):
-    typecode    = 'o'
+    typecode = 'o'
 
     def _unwrap_object(self, space, w_obj):
         try:
@@ -574,36 +571,41 @@
         address[0] = rffi.cast(rffi.VOIDP, self._unwrap_object(space, w_value))
 
 class InstancePtrPtrConverter(InstancePtrConverter):
-    _immutable_fields_ = ['uses_local']
+    typecode = 'o'
 
-    uses_local = True
+    def __init__(self, space, extra):
+        InstancePtrConverter.__init__(self, space, extra)
+        self.ref_buffer = lltype.nullptr(rffi.VOIDPP.TO)
 
-    def convert_argument(self, space, w_obj, address, call_local):
-        r = rffi.cast(rffi.VOIDPP, call_local)
-        r[0] = rffi.cast(rffi.VOIDP, self._unwrap_object(space, w_obj))
+    def convert_argument(self, space, w_obj, address):
         x = rffi.cast(rffi.VOIDPP, address)
-        x[0] = rffi.cast(rffi.VOIDP, call_local)
-        address = rffi.cast(capi.C_OBJECT, address)
+        ptr = rffi.cast(rffi.VOIDP, self._unwrap_object(space, w_obj))
+        self.ref_buffer = lltype.malloc(rffi.VOIDPP.TO, 1, flavor='raw')
+        self.ref_buffer[0] = ptr
+        x[0] = self.ref_buffer
         ba = rffi.cast(rffi.CCHARP, address)
-        ba[capi.c_function_arg_typeoffset(space)] = 'o'
+        ba[capi.c_function_arg_typeoffset(space)] = self.typecode
 
-    def convert_argument_libffi(self, space, w_obj, address, call_local):
+    def convert_argument_libffi(self, space, w_obj, address, scratch):
         # TODO: finalize_call not yet called for fast call (see interp_cppyy.py)
         from pypy.module._cppyy.interp_cppyy import FastCallNotPossible
         raise FastCallNotPossible
 
-    def finalize_call(self, space, w_obj, call_local):
-        from pypy.module._cppyy.interp_cppyy import W_CPPInstance
-        assert isinstance(w_obj, W_CPPInstance)
-        r = rffi.cast(rffi.VOIDPP, call_local)
-        w_obj._rawobject = rffi.cast(capi.C_OBJECT, r[0])
-
     def from_memory(self, space, w_obj, w_pycppclass, offset):
         address = rffi.cast(capi.C_OBJECT, self._get_raw_address(space, w_obj, offset))
         from pypy.module._cppyy import interp_cppyy
         return interp_cppyy.wrap_cppinstance(
             space, address, self.clsdecl, do_cast=False, is_ref=True)
 
+    def finalize_call(self, space, w_obj):
+        if self.ref_buffer:
+            set_rawobject(space, w_obj, self.ref_buffer[0])
+
+    def free_argument(self, space, arg):
+        if self.ref_buffer:
+            lltype.free(self.ref_buffer, flavor='raw')
+            self.ref_buffer = lltype.nullptr(rffi.VOIDPP.TO)
+
 class StdStringConverter(InstanceConverter):
 
     def __init__(self, space, extra):
@@ -628,7 +630,7 @@
         except Exception:
             InstanceConverter.to_memory(self, space, w_obj, w_value, offset)
 
-    def free_argument(self, space, arg, call_local):
+    def free_argument(self, space, arg):
         capi.c_destruct(space, self.clsdecl, rffi.cast(capi.C_OBJECT, rffi.cast(rffi.VOIDPP, arg)[0]))
 
 class StdStringRefConverter(InstancePtrConverter):
@@ -646,7 +648,7 @@
         state = space.fromcache(ffitypes.State)
         return state.c_voidp
 
-    def convert_argument(self, space, w_obj, address, call_local):
+    def convert_argument(self, space, w_obj, address):
         if hasattr(space, "fake"):
             raise NotImplementedError
         space.getbuiltinmodule("cpyext")
@@ -657,7 +659,7 @@
         ba = rffi.cast(rffi.CCHARP, address)
         ba[capi.c_function_arg_typeoffset(space)] = 'a'
 
-    def convert_argument_libffi(self, space, w_obj, address, call_local):
+    def convert_argument_libffi(self, space, w_obj, address, scratch):
         # TODO: free_argument not yet called for fast call (see interp_cppyy.py)
         from pypy.module._cppyy.interp_cppyy import FastCallNotPossible
         raise FastCallNotPossible
@@ -671,7 +673,7 @@
         x = rffi.cast(rffi.VOIDPP, address)
         x[0] = rffi.cast(rffi.VOIDP, ref)"""
 
-    def free_argument(self, space, arg, call_local):
+    def free_argument(self, space, arg):
         if hasattr(space, "fake"):
             raise NotImplementedError
         space.getbuiltinmodule("cpyext")
@@ -685,7 +687,7 @@
     def __init__(self, space, signature):
         self.signature = signature
 
-    def convert_argument(self, space, w_obj, address, call_local):
+    def convert_argument(self, space, w_obj, address):
         # TODO: atm, does not actually get an overload, but a staticmethod
         from pypy.module._cppyy.interp_cppyy import W_CPPOverload
         cppol = space.interp_w(W_CPPOverload, w_obj)
@@ -740,7 +742,7 @@
         raise oefmt(space.w_TypeError,
                     "cannot pass %T instance as %s", w_obj, self.rawdecl.name)
 
-    def convert_argument(self, space, w_obj, address, call_local):
+    def convert_argument(self, space, w_obj, address):
         x = rffi.cast(rffi.VOIDPP, address)
         x[0] = rffi.cast(rffi.VOIDP, self._unwrap_object(space, w_obj))
         address = rffi.cast(capi.C_OBJECT, address)
diff --git a/pypy/module/_cppyy/interp_cppyy.py b/pypy/module/_cppyy/interp_cppyy.py
--- a/pypy/module/_cppyy/interp_cppyy.py
+++ b/pypy/module/_cppyy/interp_cppyy.py
@@ -1,6 +1,9 @@
 import pypy.module._cppyy.capi as capi
 
 from pypy.interpreter.error import OperationError, oefmt
+from pypy.interpreter.function import Method
+from pypy.interpreter.argument import Arguments
+from pypy.interpreter.typedef import interp_attrproperty_w, descr_generic_ne, make_weakref_descr
 from pypy.interpreter.gateway import interp2app, unwrap_spec
 from pypy.interpreter.typedef import TypeDef, GetSetProperty, interp_attrproperty
 from pypy.interpreter.baseobjspace import W_Root
@@ -166,11 +169,13 @@
 #  W_CPPConstructorOverload:      constructors
 #  W_CPPStaticOverload:           free and static functions
 #  W_CPPTemplateOverload:         templated methods
-#  W_CPPTemplateStaticOveload:    templated free and static functions
+#  W_CPPTemplateStaticOverload:   templated free and static functions
 #
 #  CPPMethod:         a single function or method (base class)
 #  CPPSetItem:        specialization for Python's __setitem__
 #
+#  MethodWithProps:   python instancemethod that forwards properties
+#
 # All methods/functions derive from CPPMethod and are collected as overload
 # candidates in user-facing overload classes. Templated methods are a two-step
 # process, where first the template is instantiated (or selected if already
@@ -183,13 +188,13 @@
     the memory_regulator."""
 
     _attrs_ = ['space', 'scope', 'cppmethod', 'arg_defs', 'args_required',
-               'converters', 'executor', '_funcaddr', 'cif_descr', 'uses_local']
+               'converters', 'executor', '_funcaddr', 'cif_descr']
     _immutable_fields_ = ['scope', 'cppmethod', 'arg_defs', 'args_required',
-                          'converters', 'executor', 'uses_local']
+               'converters', 'executor', '_funcaddr', 'cif_descr']
 
-    def __init__(self, space, declaring_scope, cppmethod, arg_defs, args_required):
+    def __init__(self, space, decl_scope, cppmethod, arg_defs, args_required):
         self.space = space
-        self.scope = declaring_scope
+        self.scope = decl_scope
         self.cppmethod = cppmethod
         self.arg_defs = arg_defs
         self.args_required = args_required
@@ -200,14 +205,6 @@
         self.executor = None
         self.cif_descr = lltype.nullptr(jit_libffi.CIF_DESCRIPTION)
         self._funcaddr = lltype.nullptr(capi.C_FUNC_PTR.TO)
-        self.uses_local = False
-
-    def _address_from_local_buffer(self, call_local, idx):
-        if not call_local:
-            return call_local
-        stride = 2*rffi.sizeof(rffi.VOIDP)
-        loc_idx = lltype.direct_ptradd(rffi.cast(rffi.CCHARP, call_local), idx*stride)
-        return rffi.cast(rffi.VOIDP, loc_idx)
 
     @jit.unroll_safe
     def call(self, cppthis, args_w, useffi):
@@ -233,45 +230,35 @@
             except Exception:
                 pass
 
-        # some calls, e.g. for ptr-ptr or reference need a local array to store data for
-        # the duration of the call
-        if self.uses_local:
-            call_local = lltype.malloc(rffi.VOIDP.TO, 2*len(args_w), flavor='raw')
-        else:
-            call_local = lltype.nullptr(rffi.VOIDP.TO)
+        # attempt to call directly through ffi chain
+        if useffi and self._funcaddr:
+            try:
+                return self.do_fast_call(cppthis, args_w)
+            except FastCallNotPossible:
+                pass      # can happen if converters or executor does not implement ffi
 
+        # ffi chain must have failed; using stub functions instead
+        args, stat = self.prepare_arguments(args_w)
         try:
-            # attempt to call directly through ffi chain
-            if useffi and self._funcaddr:
-                try:
-                    return self.do_fast_call(cppthis, args_w, call_local)
-                except FastCallNotPossible:
-                    pass      # can happen if converters or executor does not implement ffi
-
-            # ffi chain must have failed; using stub functions instead
-            args, stat = self.prepare_arguments(args_w, call_local)
-            try:
-                result = self.executor.execute(
-                    self.space, self.cppmethod, cppthis, len(args_w), args)
-                if stat[0] != rffi.cast(rffi.ULONG, 0):
-                    what = rffi.cast(rffi.CCHARP, stat[1])
-                    pywhat = rffi.charp2str(what)
-                    capi.c_free(self.space, rffi.cast(rffi.VOIDP, what))
-                    raise OperationError(self.space.w_Exception, self.space.newtext(pywhat))
-                return result
-            finally:
-                self.finalize_call(args, args_w, call_local)
+            result = self.executor.execute(
+                self.space, self.cppmethod, cppthis, len(args_w), args)
+            if stat[0] != rffi.cast(rffi.ULONG, 0):
+                what = rffi.cast(rffi.CCHARP, stat[1])
+                pywhat = rffi.charp2str(what)
+                capi.c_free(self.space, rffi.cast(rffi.VOIDP, what))
+                raise OperationError(self.space.w_Exception, self.space.newtext(pywhat))
+            return result
         finally:
-            if call_local:
-                lltype.free(call_local, flavor='raw')
+            self.finalize_call(args, args_w)
 
     @jit.unroll_safe
-    def do_fast_call(self, cppthis, args_w, call_local):
+    def do_fast_call(self, cppthis, args_w):
         if self.cif_descr == lltype.nullptr(jit_libffi.CIF_DESCRIPTION):
             raise FastCallNotPossible
         jit.promote(self)
         cif_descr = self.cif_descr
-        buffer = lltype.malloc(rffi.CCHARP.TO, cif_descr.exchange_size, flavor='raw')
+        # add extra space for const-ref support (see converter.py)
+        buffer = lltype.malloc(rffi.CCHARP.TO, cif_descr.exchange_size+len(args_w)*rffi.sizeof(rffi.DOUBLE), flavor='raw')
         try:
             # this pointer
             data = rffi.ptradd(buffer, cif_descr.exchange_args[0])
@@ -284,7 +271,8 @@
                 conv = self.converters[i]
                 w_arg = args_w[i]
                 data = rffi.ptradd(buffer, cif_descr.exchange_args[i+1])
-                conv.convert_argument_libffi(self.space, w_arg, data, call_local)
+                scratch = rffi.ptradd(buffer, cif_descr.exchange_size+i*rffi.sizeof(rffi.DOUBLE))
+                conv.convert_argument_libffi(self.space, w_arg, data, scratch)
             for j in range(i+1, len(self.arg_defs)):
                 conv = self.converters[j]
                 data = rffi.ptradd(buffer, cif_descr.exchange_args[j+1])
@@ -346,11 +334,6 @@
         self.executor = executor.get_executor(
             self.space, capi.c_method_result_type(self.space, self.cppmethod))
 
-        for conv in self.converters:
-            if conv.uses_local:
-                self.uses_local = True
-                break
-
         # Each CPPMethod corresponds one-to-one to a C++ equivalent and cppthis
         # has been offset to the matching class. Hence, the libffi pointer is
         # uniquely defined and needs to be setup only once.
@@ -386,7 +369,7 @@
             self._funcaddr = funcaddr
 
     @jit.unroll_safe
-    def prepare_arguments(self, args_w, call_local):
+    def prepare_arguments(self, args_w):
         args = capi.c_allocate_function_args(self.space, len(args_w))
         stride = capi.c_function_arg_sizeof(self.space)
         for i in range(len(args_w)):
@@ -394,15 +377,13 @@
             w_arg = args_w[i]
             try:
                 arg_i = lltype.direct_ptradd(rffi.cast(rffi.CCHARP, args), i*stride)
-                loc_i = self._address_from_local_buffer(call_local, i)
-                conv.convert_argument(self.space, w_arg, rffi.cast(capi.C_OBJECT, arg_i), loc_i)
+                conv.convert_argument(self.space, w_arg, rffi.cast(capi.C_OBJECT, arg_i))
             except:
                 # fun :-(
                 for j in range(i):
                     conv = self.converters[j]
                     arg_j = lltype.direct_ptradd(rffi.cast(rffi.CCHARP, args), j*stride)
-                    loc_j = self._address_from_local_buffer(call_local, j)
-                    conv.free_argument(self.space, rffi.cast(capi.C_OBJECT, arg_j), loc_j)
+                    conv.free_argument(self.space, rffi.cast(capi.C_OBJECT, arg_j))
                 capi.c_deallocate_function_args(self.space, args)
                 raise
         stat = rffi.cast(rffi.ULONGP,
@@ -411,14 +392,13 @@
         return args, stat
 
     @jit.unroll_safe
-    def finalize_call(self, args, args_w, call_local):
+    def finalize_call(self, args, args_w):
         stride = capi.c_function_arg_sizeof(self.space)
         for i in range(len(args_w)):
             conv = self.converters[i]
             arg_i = lltype.direct_ptradd(rffi.cast(rffi.CCHARP, args), i*stride)
-            loc_i = self._address_from_local_buffer(call_local, i)
-            conv.finalize_call(self.space, args_w[i], loc_i)
-            conv.free_argument(self.space, rffi.cast(capi.C_OBJECT, arg_i), loc_i)
+            conv.finalize_call(self.space, args_w[i])
+            conv.free_argument(self.space, rffi.cast(capi.C_OBJECT, arg_i))
         capi.c_deallocate_function_args(self.space, args)
 
     def signature(self, show_formalargs=True):
@@ -466,42 +446,81 @@
         CPPMethod.call(self, cppthis, args_w, useffi)
 
 
+# CPPOverloads have settable flags that control memory and ffi behavior. These flags
+# need forwarding, which the normal instancemethod does not provide, hence this
+# derived class.
+class MethodWithProps(Method):
+    # allow user to determine ffi use rules per overload
+    def fget_useffi(self, space):
+	f = space.interp_w(W_CPPOverload, self.w_function)
+        return f.fget_useffi(space)
+
+    @unwrap_spec(value=bool)
+    def fset_useffi(self, space, value):
+        f = space.interp_w(W_CPPOverload, self.w_function)
+        f.fset_useffi(space, value)
+
+MethodWithProps.typedef = TypeDef(
+    "cpp_instancemethod",
+    __doc__ = """cpp_instancemethod(function, instance, class)
+
+Create an instance method object.""",
+    __new__ = interp2app(MethodWithProps.descr_method__new__.im_func),
+    __call__ = interp2app(MethodWithProps.descr_method_call),
+    __get__ = interp2app(MethodWithProps.descr_method_get),
+    im_func = interp_attrproperty_w('w_function', cls=MethodWithProps),
+    __func__ = interp_attrproperty_w('w_function', cls=MethodWithProps),
+    im_self = interp_attrproperty_w('w_instance', cls=MethodWithProps),
+    __self__ = interp_attrproperty_w('w_instance', cls=MethodWithProps),
+    im_class = interp_attrproperty_w('w_class', cls=MethodWithProps),
+    __getattribute__ = interp2app(MethodWithProps.descr_method_getattribute),
+    __eq__ = interp2app(MethodWithProps.descr_method_eq),
+    __ne__ = descr_generic_ne,
+    __hash__ = interp2app(MethodWithProps.descr_method_hash),
+    __repr__ = interp2app(MethodWithProps.descr_method_repr),
+    __reduce__ = interp2app(MethodWithProps.descr_method__reduce__),
+    __weakref__ = make_weakref_descr(MethodWithProps),
+    __useffi__ = GetSetProperty(MethodWithProps.fget_useffi, MethodWithProps.fset_useffi),
+    )
+MethodWithProps.typedef.acceptable_as_base_class = False
+
+
 class W_CPPOverload(W_Root):
     """App-level dispatcher: controls a collection of (potentially) overloaded methods
     or functions. Calls these in order and deals with error handling and reporting."""
 
-    _attrs_ = ['space', 'scope', 'functions', 'flags', 'w_this']
+    _attrs_ = ['space', 'scope', 'functions', 'flags']
     _immutable_fields_ = ['scope', 'functions[*]']
 
-    def __init__(self, space, declaring_scope, functions, flags = OVERLOAD_FLAGS_USE_FFI):
+    def __init__(self, space, decl_scope, funcs, flags = OVERLOAD_FLAGS_USE_FFI):
         self.space  = space
-        self.scope  = declaring_scope
+        self.scope  = decl_scope
         from rpython.rlib import debug
-        self.functions = debug.make_sure_not_resized(functions)
+        self.functions = debug.make_sure_not_resized(funcs)
         self.flags  = flags
-        self.w_this = self.space.w_None
+
+    def descr_get(self, w_obj, w_cls=None):
+        """functionobject.__get__(obj[, type]) -> method"""
+        # TODO: check validity of w_cls if given
+        space = self.space
+        asking_for_bound = (space.is_none(w_cls) or
+                            not space.is_w(w_obj, space.w_None) or
+                            space.is_w(w_cls, space.type(space.w_None)))
+        if asking_for_bound:
+            return MethodWithProps(space, self, w_obj, w_cls)
+        else:
+            return MethodWithProps(space, self, None, w_cls)
 
     @unwrap_spec(args_w='args_w')
-    def descr_get(self, w_cppinstance, args_w):
-        if self.space.is_w(w_cppinstance, self.space.w_None):
-            return self  # unbound, so no new instance needed
-        cppol = W_CPPOverload(self.space, self.scope, self.functions, self.flags)
-        cppol.w_this = w_cppinstance
-        return cppol     # bound
-
-    @unwrap_spec(args_w='args_w')
-    def call(self, args_w):
-        if self.space.is_w(self.w_this, self.space.w_None) and len(args_w):
-            w_this = args_w[0]
-            args_w = args_w[1:]
-        else:
-            w_this = self.w_this
+    def call_args(self, args_w):
+        jit.promote(self)
+        w_this = args_w[0]
         cppinstance = self.space.interp_w(W_CPPInstance, w_this)
         cppinstance._nullcheck()
         if not capi.c_is_subtype(self.space, cppinstance.clsdecl, self.scope):
             raise oefmt(self.space.w_TypeError,
                 "cannot pass %T instance as %s", w_this, self.scope.name)
-        return self.call_impl(cppinstance.get_cppthis(self.scope), args_w)
+        return self.call_impl(cppinstance.get_cppthis(self.scope), args_w[1:])
 
     @jit.unroll_safe
     def call_impl(self, cppthis, args_w):
@@ -576,13 +595,17 @@
     def fget_doc(self, space):
         return self.prototype()
 
+    def getname(self, space): 
+        # for the benefit of Method/instancemethod
+        return capi.c_method_name(space, self.functions[0].cppmethod)
+
     def __repr__(self):
         return "W_CPPOverload(%s)" % [f.prototype() for f in self.functions]
 
 W_CPPOverload.typedef = TypeDef(
     'CPPOverload',
     __get__    = interp2app(W_CPPOverload.descr_get),
-    __call__   = interp2app(W_CPPOverload.call),
+    __call__   = interp2app(W_CPPOverload.call_args),
     __useffi__ = GetSetProperty(W_CPPOverload.fget_useffi, W_CPPOverload.fset_useffi),
     __doc__    = GetSetProperty(W_CPPOverload.fget_doc)
 )
@@ -593,27 +616,24 @@
 class W_CPPStaticOverload(W_CPPOverload):
     _attrs_ = []
 
-    @unwrap_spec(args_w='args_w')
-    def descr_get(self, w_cppinstance, args_w):
-        if isinstance(w_cppinstance, W_CPPInstance):
+    def descr_get(self, w_obj, w_cls=None):
+        if isinstance(w_obj, W_CPPInstance):
             # two possibilities: this is a static function called on an
             # instance and w_this must not be set, or a free function rebound
             # onto a class and w_this should be set
-            cppinstance = self.space.interp_w(W_CPPInstance, w_cppinstance)
+            cppinstance = self.space.interp_w(W_CPPInstance, w_obj)
             if cppinstance.clsdecl.handle != self.scope.handle:
-                cppol = W_CPPStaticOverload(self.space, self.scope, self.functions, self.flags)
-                cppol.w_this = w_cppinstance
-                return cppol       # bound
+	        return MethodWithProps(self.space, self, w_obj, w_cls)    # bound
         return self      # unbound
 
     @unwrap_spec(args_w='args_w')
-    def call(self, args_w):
-        if not self.space.is_w(self.w_this, self.space.w_None):
-            # free function used as bound method, put self back into args_w
-            cppinstance = self.space.interp_w(W_CPPInstance, self.w_this)
-            cppinstance._nullcheck()
-            args_w = [self.w_this] + args_w
+    def call_args(self, args_w):
+        jit.promote(self)
+        #if isinstance(args_w[0], W_CPPInstance):
+            # free function used as bound method, leave in place
         return self.call_impl(capi.C_NULL_OBJECT, args_w)
+        # free functions are implemented as methods of 'namespace' classes, remove 'instance'
+        #return self.call_impl(capi.C_NULL_OBJECT, args_w[1:])
 
     def __repr__(self):
         return "W_CPPStaticOverload(%s)" % [f.prototype() for f in self.functions]
@@ -621,7 +641,7 @@
 W_CPPStaticOverload.typedef = TypeDef(
     'CPPStaticOverload',
     __get__    = interp2app(W_CPPStaticOverload.descr_get),
-    __call__   = interp2app(W_CPPStaticOverload.call),
+    __call__   = interp2app(W_CPPStaticOverload.call_args),
     __useffi__ = GetSetProperty(W_CPPStaticOverload.fget_useffi, W_CPPStaticOverload.fset_useffi),
     __doc__    = GetSetProperty(W_CPPStaticOverload.fget_doc)
 )
@@ -631,26 +651,15 @@
     _attrs_ = []
 
     @unwrap_spec(args_w='args_w')
-    def descr_get(self, w_cppinstance, args_w):
-        if self.space.is_w(w_cppinstance, self.space.w_None):
-            return self  # unbound (TODO: probably useless)
-        cppol = W_CPPConstructorOverload(self.space, self.scope, self.functions, self.flags)
-        cppol.w_this = w_cppinstance
-        return cppol     # bound
-
-    @unwrap_spec(args_w='args_w')
-    def call(self, args_w):
+    def call_args(self, args_w):
+        jit.promote(self)
         # TODO: factor out the following:
         if capi.c_is_abstract(self.space, self.scope.handle):
             raise oefmt(self.space.w_TypeError,
                         "cannot instantiate abstract class '%s'",
                         self.scope.name)
-        if self.space.is_w(self.w_this, self.space.w_None) and len(args_w):
-            cppinstance = self.space.interp_w(W_CPPInstance, args_w[0])
-            args_w = args_w[1:]
-        else:
-            cppinstance = self.space.interp_w(W_CPPInstance, self.w_this)
-        w_result = self.call_impl(rffi.cast(capi.C_OBJECT, self.scope.handle), args_w)
+        cppinstance = self.space.interp_w(W_CPPInstance, args_w[0])
+        w_result = self.call_impl(rffi.cast(capi.C_OBJECT, self.scope.handle), args_w[1:])
         newthis = rffi.cast(capi.C_OBJECT, self.space.uint_w(w_result))
         if cppinstance is not None:
             cppinstance._rawobject = newthis
@@ -662,7 +671,7 @@
 W_CPPConstructorOverload.typedef = TypeDef(
     'CPPConstructorOverload',
     __get__    = interp2app(W_CPPConstructorOverload.descr_get),
-    __call__   = interp2app(W_CPPConstructorOverload.call),
+    __call__   = interp2app(W_CPPConstructorOverload.call_args),
     __doc__    = GetSetProperty(W_CPPConstructorOverload.fget_doc)
 )
 
@@ -725,7 +734,9 @@
         # try to match with run-time instantiations
         for cppol in self.master.overloads.values():
             try:
-                return cppol.descr_get(self.w_this, []).call(args_w)
+                if not self.space.is_w(self.w_this, self.space.w_None):
+                    return self.space.call_obj_args(cppol, self.w_this, Arguments(self.space, args_w))
+                return self.space.call_args(cppol, Arguments(self.space, args_w))
             except Exception:
                 pass    # completely ignore for now; have to see whether errors become confusing
 
@@ -748,7 +759,9 @@
             except KeyError:
                 self.master.overloads[fullname] = method
 
-        return method.descr_get(self.w_this, []).call(args_w)
+        if not self.space.is_w(self.w_this, self.space.w_None):
+	    return self.space.call_obj_args(method, self.w_this, Arguments(self.space, args_w))
+        return self.space.call_args(method, Arguments(self.space, args_w))
 
     def getitem_impl(self, name, args_w):
         space = self.space
@@ -771,25 +784,25 @@
             if c_fullname != fullname:
                 self.master.overloads[c_fullname] = method
 
-        return method.descr_get(self.w_this, [])
+        return method.descr_get(self.w_this, None)
 
 
 class W_CPPTemplateOverload(W_CPPOverload, TemplateOverloadMixin):
     """App-level dispatcher to allow both lookup/instantiation of templated methods and
     dispatch among overloads between templated and non-templated method."""
 
-    _attrs_ = ['name', 'overloads', 'master']
+    _attrs_ = ['name', 'overloads', 'master', 'w_this']
     _immutable_fields_ = ['name']
 
-    def __init__(self, space, name, declaring_scope, functions, flags = OVERLOAD_FLAGS_USE_FFI):
-         W_CPPOverload.__init__(self, space, declaring_scope, functions, flags)
+    def __init__(self, space, name, decl_scope, functions, flags = OVERLOAD_FLAGS_USE_FFI):
+         W_CPPOverload.__init__(self, space, decl_scope, functions, flags)
          self.name = name
          self.overloads = {}
          self.master = self
+	 self.w_this = space.w_None
 
-    @unwrap_spec(args_w='args_w')
-    def descr_get(self, w_cppinstance, args_w):
-        # like W_CPPOverload, but returns W_CPPTemplateOverload
+    def descr_get(self, w_cppinstance, w_cls=None):
+	# TODO: don't return copy, but bind in an external object (like W_CPPOverload)
         if self.space.is_w(w_cppinstance, self.space.w_None):
             return self  # unbound, so no new instance needed
         cppol = W_CPPTemplateOverload(self.space, self.name, self.scope, self.functions, self.flags)
@@ -798,13 +811,13 @@
         return cppol     # bound
 
     @unwrap_spec(args_w='args_w')
-    def call(self, args_w):
+    def call_args(self, args_w):
         # direct call: either pick non-templated overload or attempt to deduce
         # the template instantiation from the argument types
 
         # try existing overloads or compile-time instantiations
         try:
-            return W_CPPOverload.call(self, args_w)
+            return W_CPPOverload.call_args(self, args_w)
         except Exception:
             pass
 
@@ -814,6 +827,9 @@
     def getitem(self, args_w):
         return self.getitem_impl(self.name, args_w)
 
+    def getname(self, space):
+        return self.name
+
     def __repr__(self):
         return "W_CPPTemplateOverload(%s)" % [f.prototype() for f in self.functions]
 
@@ -821,7 +837,7 @@
     'CPPTemplateOverload',
     __get__     = interp2app(W_CPPTemplateOverload.descr_get),
     __getitem__ = interp2app(W_CPPTemplateOverload.getitem),
-    __call__    = interp2app(W_CPPTemplateOverload.call),
+    __call__    = interp2app(W_CPPTemplateOverload.call_args),
     __useffi__  = GetSetProperty(W_CPPTemplateOverload.fget_useffi, W_CPPTemplateOverload.fset_useffi),
     __doc__     = GetSetProperty(W_CPPTemplateOverload.fget_doc)
 )
@@ -830,18 +846,18 @@
     """App-level dispatcher to allow both lookup/instantiation of templated methods and
     dispatch among overloads between templated and non-templated method."""
 
-    _attrs_ = ['name', 'overloads', 'master']
+    _attrs_ = ['name', 'overloads', 'master', 'w_this']
     _immutable_fields_ = ['name']
 
-    def __init__(self, space, name, declaring_scope, functions, flags = OVERLOAD_FLAGS_USE_FFI):
-         W_CPPStaticOverload.__init__(self, space, declaring_scope, functions, flags)
+    def __init__(self, space, name, decl_scope, funcs, flags = OVERLOAD_FLAGS_USE_FFI):
+         W_CPPStaticOverload.__init__(self, space, decl_scope, funcs, flags)
          self.name = name
          self.overloads = {}
          self.master = self
+	 self.w_this = space.w_None
 
-    @unwrap_spec(args_w='args_w')
-    def descr_get(self, w_cppinstance, args_w):
-        # like W_CPPStaticOverload, but returns W_CPPTemplateStaticOverload
+    def descr_get(self, w_cppinstance, w_cls=None):
+	# TODO: don't return copy, but bind in an external object (like W_CPPOverload)
         if isinstance(w_cppinstance, W_CPPInstance):
             cppinstance = self.space.interp_w(W_CPPInstance, w_cppinstance)
             if cppinstance.clsdecl.handle != self.scope.handle:
@@ -852,13 +868,13 @@
         return self      # unbound
 
     @unwrap_spec(args_w='args_w')
-    def call(self, args_w):
+    def call_args(self, args_w):
         # direct call: either pick non-templated overload or attempt to deduce
         # the template instantiation from the argument types
 
         # try existing overloads or compile-time instantiations
         try:
-            return W_CPPStaticOverload.call(self, args_w)
+            return W_CPPStaticOverload.call_args(self, args_w)
         except Exception:
             pass
 
@@ -869,6 +885,9 @@
     def getitem(self, args_w):
         return self.getitem_impl(self.name, args_w)
 
+    def getname(self, space):
+        return self.name
+
     def __repr__(self):
         return "W_CPPTemplateStaticOverload(%s)" % [f.prototype() for f in self.functions]
 
@@ -876,7 +895,7 @@
     'CPPTemplateStaticOverload',
     __get__     = interp2app(W_CPPTemplateStaticOverload.descr_get),
     __getitem__ = interp2app(W_CPPTemplateStaticOverload.getitem),
-    __call__    = interp2app(W_CPPTemplateStaticOverload.call),
+    __call__    = interp2app(W_CPPTemplateStaticOverload.call_args),
     __useffi__  = GetSetProperty(W_CPPTemplateStaticOverload.fget_useffi, W_CPPTemplateStaticOverload.fset_useffi),
     __doc__     = GetSetProperty(W_CPPTemplateStaticOverload.fget_doc)
 )
@@ -898,9 +917,9 @@
     _attrs_ = ['space', 'scope', 'converter', 'offset']
     _immutable_fields = ['scope', 'converter', 'offset']
 
-    def __init__(self, space, declaring_scope, type_name, offset):
+    def __init__(self, space, decl_scope, type_name, offset):
         self.space = space
-        self.scope = declaring_scope
+        self.scope = decl_scope
         self.converter = converter.get_converter(self.space, type_name, '')
         self.offset = offset
 
@@ -1417,7 +1436,7 @@
                     ol = W_CPPStaticOverload(self.space, nss, funcs[:])
                     # TODO: cache this operator (not done yet, as the above does not
                     # select all overloads)
-                    return ol.call([self, w_other])
+                    return ol.call_args([self, w_other])
         except OperationError as e:
             if not e.match(self.space, self.space.w_TypeError):
                 raise
diff --git a/pypy/module/_cppyy/test/test_zjit.py b/pypy/module/_cppyy/test/test_zjit.py
--- a/pypy/module/_cppyy/test/test_zjit.py
+++ b/pypy/module/_cppyy/test/test_zjit.py
@@ -205,6 +205,9 @@
     def exception_match(self, typ, sub):
         return typ is sub
 
+    def is_none(self, w_obj):
+        return w_obj is None
+
     def is_w(self, w_one, w_two):
         return w_one is w_two
 
@@ -268,6 +271,9 @@
     def call_function(self, w_func, *args_w):
         return None
 
+    def call_obj_args(self, w_callable, w_obj, args):
+        return w_callable.call_args([w_obj]+args)
+
     def _freeze_(self):
         return True
 
@@ -283,19 +289,19 @@
         def f():
             cls  = interp_cppyy.scope_byname(space, "example01")
             inst = interp_cppyy._bind_object(space, FakeInt(0), cls, True)
-            cls.get_overload("__init__").descr_get(inst, []).call([FakeInt(0)])
+            cls.get_overload("__init__").descr_get(inst, []).call_args([FakeInt(0)])
             cppmethod = cls.get_overload(method_name)
             assert isinstance(inst, interp_cppyy.W_CPPInstance)
             i = 10
             while i > 0:
                 drv.jit_merge_point(inst=inst, cppmethod=cppmethod, i=i)
-                cppmethod.descr_get(inst, []).call([FakeInt(i)])
+                cppmethod.descr_get(inst, []).call_args([FakeInt(i)])
                 i -= 1
             return 7
         f()
         space = FakeSpace()
         result = self.meta_interp(f, [], listops=True, backendopt=True, listcomp=True)
-        self.check_jitcell_token_count(0)   # same for fast and slow path??
+        self.check_jitcell_token_count(1)
         # rely on replacement of capi calls to raise exception instead (see FakeSpace.__init__)
 
     @py.test.mark.dont_track_allocations("cppmethod.cif_descr kept 'leaks'")

From pypy.commits at gmail.com  Fri Jul  6 01:34:47 2018
From: pypy.commits at gmail.com (wlav)
Date: Thu, 05 Jul 2018 22:34:47 -0700 (PDT)
Subject: [pypy-commit] pypy cppyy-packaging: enable fast ffi path for
 free/static functions
Message-ID: <5b3eff77.1c69fb81.ee3ba.4c32@mx.google.com>

Author: Wim Lavrijsen <WLavrijsen at lbl.gov>
Branch: cppyy-packaging
Changeset: r94812:883089c1c89e
Date: 2018-07-05 22:15 -0700
http://bitbucket.org/pypy/pypy/changeset/883089c1c89e/

Log:	enable fast ffi path for free/static functions

diff --git a/pypy/module/_cppyy/interp_cppyy.py b/pypy/module/_cppyy/interp_cppyy.py
--- a/pypy/module/_cppyy/interp_cppyy.py
+++ b/pypy/module/_cppyy/interp_cppyy.py
@@ -258,24 +258,27 @@
         jit.promote(self)
         cif_descr = self.cif_descr
         # add extra space for const-ref support (see converter.py)
-        buffer = lltype.malloc(rffi.CCHARP.TO, cif_descr.exchange_size+len(args_w)*rffi.sizeof(rffi.DOUBLE), flavor='raw')
+        buffer = lltype.malloc(rffi.CCHARP.TO, cif_descr.exchange_size+len(self.arg_defs)*rffi.sizeof(rffi.DOUBLE), flavor='raw')
+        thisoff = 0
         try:
-            # this pointer
-            data = rffi.ptradd(buffer, cif_descr.exchange_args[0])
-            x = rffi.cast(rffi.LONGP, data)       # LONGP needed for test_zjit.py
-            x[0] = rffi.cast(rffi.LONG, cppthis)
+            if cppthis:
+                # this pointer
+                data = rffi.ptradd(buffer, cif_descr.exchange_args[0])
+                x = rffi.cast(rffi.LONGP, data)       # LONGP needed for test_zjit.py
+                x[0] = rffi.cast(rffi.LONG, cppthis)
+                thisoff = 1
 
-            # other arguments and defaults
-            i = len(self.arg_defs) + 1
+            # actual provided arguments
+            i = -1      # needed if all arguments are defaults
             for i in range(len(args_w)):
                 conv = self.converters[i]
-                w_arg = args_w[i]
-                data = rffi.ptradd(buffer, cif_descr.exchange_args[i+1])
+                data = rffi.ptradd(buffer, cif_descr.exchange_args[i+thisoff])
                 scratch = rffi.ptradd(buffer, cif_descr.exchange_size+i*rffi.sizeof(rffi.DOUBLE))
-                conv.convert_argument_libffi(self.space, w_arg, data, scratch)
+                conv.convert_argument_libffi(self.space, args_w[i], data, scratch)
+            # drop in defaults for the rest
             for j in range(i+1, len(self.arg_defs)):
                 conv = self.converters[j]
-                data = rffi.ptradd(buffer, cif_descr.exchange_args[j+1])
+                data = rffi.ptradd(buffer, cif_descr.exchange_args[j+thisoff])
                 conv.default_argument_libffi(self.space, data)
 
             assert self._funcaddr
@@ -338,13 +341,13 @@
         # has been offset to the matching class. Hence, the libffi pointer is
         # uniquely defined and needs to be setup only once.
         funcaddr = capi.c_function_address(self.space, self.cppmethod)
-        if funcaddr and cppthis:      # TODO: methods only for now
+        if funcaddr:
             state = self.space.fromcache(ffitypes.State)
 
-            # argument type specification (incl. cppthis)
+            # argument type specification (incl. cppthis if applicable)
             fargs = []
             try:
-                fargs.append(state.c_voidp)
+                if cppthis: fargs.append(state.c_voidp)
                 for i, conv in enumerate(self.converters):
                     fargs.append(conv.cffi_type(self.space))
                 fresult = self.executor.cffi_type(self.space)
@@ -650,6 +653,10 @@
 class W_CPPConstructorOverload(W_CPPOverload):
     _attrs_ = []
 
+    def __init__(self, space, decl_scope, funcs, flags = OVERLOAD_FLAGS_USE_FFI):
+        W_CPPOverload.__init__(self, space, decl_scope, funcs, flags)
+        self.flags &= ~OVERLOAD_FLAGS_USE_FFI
+
     @unwrap_spec(args_w='args_w')
     def call_args(self, args_w):
         jit.promote(self)

From pypy.commits at gmail.com  Fri Jul  6 01:36:01 2018
From: pypy.commits at gmail.com (wlav)
Date: Thu, 05 Jul 2018 22:36:01 -0700 (PDT)
Subject: [pypy-commit] pypy default: enable fast ffi path for free / static
 functions
Message-ID: <5b3effc1.1c69fb81.875b.5075@mx.google.com>

Author: Wim Lavrijsen <WLavrijsen at lbl.gov>
Branch: 
Changeset: r94813:544d787e095e
Date: 2018-07-05 22:16 -0700
http://bitbucket.org/pypy/pypy/changeset/544d787e095e/

Log:	enable fast ffi path for free / static functions

diff --git a/pypy/module/_cppyy/interp_cppyy.py b/pypy/module/_cppyy/interp_cppyy.py
--- a/pypy/module/_cppyy/interp_cppyy.py
+++ b/pypy/module/_cppyy/interp_cppyy.py
@@ -258,24 +258,27 @@
         jit.promote(self)
         cif_descr = self.cif_descr
         # add extra space for const-ref support (see converter.py)
-        buffer = lltype.malloc(rffi.CCHARP.TO, cif_descr.exchange_size+len(args_w)*rffi.sizeof(rffi.DOUBLE), flavor='raw')
+        buffer = lltype.malloc(rffi.CCHARP.TO, cif_descr.exchange_size+len(self.arg_defs)*rffi.sizeof(rffi.DOUBLE), flavor='raw')
+        thisoff = 0
         try:
-            # this pointer
-            data = rffi.ptradd(buffer, cif_descr.exchange_args[0])
-            x = rffi.cast(rffi.LONGP, data)       # LONGP needed for test_zjit.py
-            x[0] = rffi.cast(rffi.LONG, cppthis)
+            if cppthis:
+                # this pointer
+                data = rffi.ptradd(buffer, cif_descr.exchange_args[0])
+                x = rffi.cast(rffi.LONGP, data)       # LONGP needed for test_zjit.py
+                x[0] = rffi.cast(rffi.LONG, cppthis)
+                thisoff = 1
 
-            # other arguments and defaults
-            i = len(self.arg_defs) + 1
+            # actual provided arguments
+            i = -1      # needed if all arguments are defaults
             for i in range(len(args_w)):
                 conv = self.converters[i]
-                w_arg = args_w[i]
-                data = rffi.ptradd(buffer, cif_descr.exchange_args[i+1])
+                data = rffi.ptradd(buffer, cif_descr.exchange_args[i+thisoff])
                 scratch = rffi.ptradd(buffer, cif_descr.exchange_size+i*rffi.sizeof(rffi.DOUBLE))
-                conv.convert_argument_libffi(self.space, w_arg, data, scratch)
+                conv.convert_argument_libffi(self.space, args_w[i], data, scratch)
+            # drop in defaults for the rest
             for j in range(i+1, len(self.arg_defs)):
                 conv = self.converters[j]
-                data = rffi.ptradd(buffer, cif_descr.exchange_args[j+1])
+                data = rffi.ptradd(buffer, cif_descr.exchange_args[j+thisoff])
                 conv.default_argument_libffi(self.space, data)
 
             assert self._funcaddr
@@ -338,13 +341,13 @@
         # has been offset to the matching class. Hence, the libffi pointer is
         # uniquely defined and needs to be setup only once.
         funcaddr = capi.c_function_address(self.space, self.cppmethod)
-        if funcaddr and cppthis:      # TODO: methods only for now
+        if funcaddr:
             state = self.space.fromcache(ffitypes.State)
 
-            # argument type specification (incl. cppthis)
+            # argument type specification (incl. cppthis if applicable)
             fargs = []
             try:
-                fargs.append(state.c_voidp)
+                if cppthis: fargs.append(state.c_voidp)
                 for i, conv in enumerate(self.converters):
                     fargs.append(conv.cffi_type(self.space))
                 fresult = self.executor.cffi_type(self.space)
@@ -650,6 +653,10 @@
 class W_CPPConstructorOverload(W_CPPOverload):
     _attrs_ = []
 
+    def __init__(self, space, decl_scope, funcs, flags = OVERLOAD_FLAGS_USE_FFI):
+        W_CPPOverload.__init__(self, space, decl_scope, funcs, flags)
+        self.flags &= ~OVERLOAD_FLAGS_USE_FFI
+
     @unwrap_spec(args_w='args_w')
     def call_args(self, args_w):
         jit.promote(self)

From pypy.commits at gmail.com  Fri Jul  6 13:00:45 2018
From: pypy.commits at gmail.com (arigo)
Date: Fri, 06 Jul 2018 10:00:45 -0700 (PDT)
Subject: [pypy-commit] pypy default: Good speed boost on
 Fraction.__hash__(). Not compatible with
Message-ID: <5b3fa03d.1c69fb81.f2279.c3f3@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r94814:7160ef7f4d63
Date: 2018-07-06 16:01 +0200
http://bitbucket.org/pypy/pypy/changeset/7160ef7f4d63/

Log:	Good speed boost on Fraction.__hash__(). Not compatible with Python
	3, needs a different hack.

diff --git a/lib-python/2.7/fractions.py b/lib-python/2.7/fractions.py
--- a/lib-python/2.7/fractions.py
+++ b/lib-python/2.7/fractions.py
@@ -517,8 +517,13 @@
             # Get integers right.
             return hash(self._numerator)
         # Expensive check, but definitely correct.
-        if self == float(self):
-            return hash(float(self))
+        # PyPy: the following 4 lines used to be almost twice slower:
+        #         if self == float(self):
+        #             return hash(float(self))
+        f = float(self)
+        x, y = f.as_integer_ratio()    # signs are correct: y is positive
+        if self._numerator == x and self._denominator == y:
+            return hash(f)
         else:
             # Use tuple's hash to avoid a high collision rate on
             # simple fractions.

From pypy.commits at gmail.com  Fri Jul  6 13:00:47 2018
From: pypy.commits at gmail.com (arigo)
Date: Fri, 06 Jul 2018 10:00:47 -0700 (PDT)
Subject: [pypy-commit] pypy default: merge heads
Message-ID: <5b3fa03f.1c69fb81.3bab4.595c@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r94815:b717d811eb2b
Date: 2018-07-06 16:26 +0200
http://bitbucket.org/pypy/pypy/changeset/b717d811eb2b/

Log:	merge heads

diff --git a/pypy/module/_cppyy/converter.py b/pypy/module/_cppyy/converter.py
--- a/pypy/module/_cppyy/converter.py
+++ b/pypy/module/_cppyy/converter.py
@@ -82,10 +82,9 @@
 
 
 class TypeConverter(object):
-    _immutable_fields_ = ['cffi_name', 'uses_local', 'name']
+    _immutable_fields_ = ['cffi_name', 'name']
 
     cffi_name  = None
-    uses_local = False
     name       = ""
 
     def __init__(self, space, extra):
@@ -108,10 +107,10 @@
         from pypy.module._cppyy.interp_cppyy import FastCallNotPossible
         raise FastCallNotPossible
 
-    def convert_argument(self, space, w_obj, address, call_local):
+    def convert_argument(self, space, w_obj, address):
         self._is_abstract(space)
 
-    def convert_argument_libffi(self, space, w_obj, address, call_local):
+    def convert_argument_libffi(self, space, w_obj, address, scratch):
         from pypy.module._cppyy.interp_cppyy import FastCallNotPossible
         raise FastCallNotPossible
 
@@ -125,10 +124,10 @@
     def to_memory(self, space, w_obj, w_value, offset):
         self._is_abstract(space)
 
-    def finalize_call(self, space, w_obj, call_local):
+    def finalize_call(self, space, w_obj):
         pass
 
-    def free_argument(self, space, arg, call_local):
+    def free_argument(self, space, arg):
         pass
 
 
@@ -172,7 +171,7 @@
         state = space.fromcache(ffitypes.State)
         return state.c_voidp
 
-    def convert_argument(self, space, w_obj, address, call_local):
+    def convert_argument(self, space, w_obj, address):
         w_tc = space.findattr(w_obj, space.newtext('typecode'))
         if w_tc is not None and space.text_w(w_tc) != self.typecode:
             raise oefmt(space.w_TypeError,
@@ -208,7 +207,7 @@
 class NumericTypeConverterMixin(object):
     _mixin_ = True
 
-    def convert_argument_libffi(self, space, w_obj, address, call_local):
+    def convert_argument_libffi(self, space, w_obj, address, scratch):
         x = rffi.cast(self.c_ptrtype, address)
         x[0] = self._unwrap_object(space, w_obj)
 
@@ -228,26 +227,23 @@
 
 class ConstRefNumericTypeConverterMixin(NumericTypeConverterMixin):
     _mixin_ = True
-    _immutable_fields_ = ['uses_local']
-
-    uses_local = True
 
     def cffi_type(self, space):
         state = space.fromcache(ffitypes.State)
         return state.c_voidp
 
-    def convert_argument_libffi(self, space, w_obj, address, call_local):
-        assert rffi.sizeof(self.c_type) <= 2*rffi.sizeof(rffi.VOIDP)  # see interp_cppyy.py
+    def convert_argument_libffi(self, space, w_obj, address, scratch):
         obj = self._unwrap_object(space, w_obj)
-        typed_buf = rffi.cast(self.c_ptrtype, call_local)
+        typed_buf = rffi.cast(self.c_ptrtype, scratch)
         typed_buf[0] = obj
         x = rffi.cast(rffi.VOIDPP, address)
-        x[0] = call_local
+        x[0] = scratch
+
 
 class IntTypeConverterMixin(NumericTypeConverterMixin):
     _mixin_ = True
 
-    def convert_argument(self, space, w_obj, address, call_local):
+    def convert_argument(self, space, w_obj, address):
         x = rffi.cast(self.c_ptrtype, address)
         x[0] = self._unwrap_object(space, w_obj)
         ba = rffi.cast(rffi.CCHARP, address)
@@ -256,7 +252,7 @@
 class FloatTypeConverterMixin(NumericTypeConverterMixin):
     _mixin_ = True
 
-    def convert_argument(self, space, w_obj, address, call_local):
+    def convert_argument(self, space, w_obj, address):
         x = rffi.cast(self.c_ptrtype, address)
         x[0] = self._unwrap_object(space, w_obj)
         ba = rffi.cast(rffi.CCHARP, address)
@@ -273,18 +269,18 @@
         state = space.fromcache(ffitypes.State)
         return state.c_void
 
-    def convert_argument(self, space, w_obj, address, call_local):
+    def convert_argument(self, space, w_obj, address):
         self._is_abstract(space)
 
 
 class BoolConverter(ffitypes.typeid(bool), TypeConverter):
-    def convert_argument(self, space, w_obj, address, call_local):
+    def convert_argument(self, space, w_obj, address):
         x = rffi.cast(rffi.LONGP, address)
         x[0] = self._unwrap_object(space, w_obj)
         ba = rffi.cast(rffi.CCHARP, address)
         ba[capi.c_function_arg_typeoffset(space)] = 'b'
 
-    def convert_argument_libffi(self, space, w_obj, address, call_local):
+    def convert_argument_libffi(self, space, w_obj, address, scratch):
         x = rffi.cast(rffi.LONGP, address)
         x[0] = self._unwrap_object(space, w_obj)
 
@@ -303,13 +299,13 @@
             address[0] = '\x00'
 
 class CharConverter(ffitypes.typeid(rffi.CHAR), TypeConverter):
-    def convert_argument(self, space, w_obj, address, call_local):
+    def convert_argument(self, space, w_obj, address):
         x = rffi.cast(rffi.CCHARP, address)
         x[0] = self._unwrap_object(space, w_obj)
         ba = rffi.cast(rffi.CCHARP, address)
         ba[capi.c_function_arg_typeoffset(space)] = 'b'
 
-    def convert_argument_libffi(self, space, w_obj, address, call_local):
+    def convert_argument_libffi(self, space, w_obj, address, scratch):
         x = rffi.cast(self.c_ptrtype, address)
         x[0] = self._unwrap_object(space, w_obj)
 
@@ -348,7 +344,7 @@
         state = space.fromcache(ffitypes.State)
         return state.c_voidp
 
-    def convert_argument_libffi(self, space, w_obj, address, call_local):
+    def convert_argument_libffi(self, space, w_obj, address, scratch):
         from pypy.module._cppyy.interp_cppyy import FastCallNotPossible
         raise FastCallNotPossible
 
@@ -381,7 +377,7 @@
 
 
 class CStringConverter(TypeConverter):
-    def convert_argument(self, space, w_obj, address, call_local):
+    def convert_argument(self, space, w_obj, address):
         x = rffi.cast(rffi.LONGP, address)
         arg = space.text_w(w_obj)
         x[0] = rffi.cast(rffi.LONG, rffi.str2charp(arg))
@@ -393,7 +389,7 @@
         charpptr = rffi.cast(rffi.CCHARPP, address)
         return space.newtext(rffi.charp2str(charpptr[0]))
 
-    def free_argument(self, space, arg, call_local):
+    def free_argument(self, space, arg):
         lltype.free(rffi.cast(rffi.CCHARPP, arg)[0], flavor='raw')
 
 class CStringConverterWithSize(CStringConverter):
@@ -423,13 +419,13 @@
         state = space.fromcache(ffitypes.State)
         return state.c_voidp
 
-    def convert_argument(self, space, w_obj, address, call_local):
+    def convert_argument(self, space, w_obj, address):
         x = rffi.cast(rffi.VOIDPP, address)
         x[0] = self._unwrap_object(space, w_obj)
         ba = rffi.cast(rffi.CCHARP, address)
         ba[capi.c_function_arg_typeoffset(space)] = 'o'
 
-    def convert_argument_libffi(self, space, w_obj, address, call_local):
+    def convert_argument_libffi(self, space, w_obj, address, scratch):
         x = rffi.cast(rffi.VOIDPP, address)
         x[0] = self._unwrap_object(space, w_obj)
 
@@ -452,37 +448,39 @@
             address[0] = rffi.cast(rffi.VOIDP, self._unwrap_object(space, w_value))
 
 class VoidPtrPtrConverter(TypeConverter):
-    _immutable_fields_ = ['uses_local', 'typecode']
+    typecode = 'p'
 
-    uses_local = True
-    typecode = 'a'
+    def __init__(self, space, extra):
+        self.ref_buffer = lltype.nullptr(rffi.VOIDPP.TO)
 
-    def convert_argument(self, space, w_obj, address, call_local):
+    def convert_argument(self, space, w_obj, address):
         x = rffi.cast(rffi.VOIDPP, address)
-        ba = rffi.cast(rffi.CCHARP, address)
         try:
             x[0] = get_rawbuffer(space, w_obj)
         except TypeError:
-            r = rffi.cast(rffi.VOIDPP, call_local)
-            r[0] = rffi.cast(rffi.VOIDP, get_rawobject(space, w_obj))
-            x[0] = rffi.cast(rffi.VOIDP, call_local)
+            ptr = rffi.cast(rffi.VOIDP, get_rawobject(space, w_obj))
+            self.ref_buffer = lltype.malloc(rffi.VOIDPP.TO, 1, flavor='raw')
+            self.ref_buffer[0] = ptr
+            x[0] = self.ref_buffer
+        ba = rffi.cast(rffi.CCHARP, address)
         ba[capi.c_function_arg_typeoffset(space)] = self.typecode
 
-    def finalize_call(self, space, w_obj, call_local):
-        r = rffi.cast(rffi.VOIDPP, call_local)
-        try:
-            set_rawobject(space, w_obj, r[0])
-        except OperationError:
-            pass             # no set on buffer/array/None
+    def finalize_call(self, space, w_obj):
+        if self.ref_buffer:
+            set_rawobject(space, w_obj, self.ref_buffer[0])
+
+    def free_argument(self, space, arg):
+        if self.ref_buffer:
+            lltype.free(self.ref_buffer, flavor='raw')
+            self.ref_buffer = lltype.nullptr(rffi.VOIDPP.TO)
 
 class VoidPtrRefConverter(VoidPtrPtrConverter):
-    _immutable_fields_ = ['uses_local', 'typecode']
-    uses_local = True
+    _immutable_fields_ = ['typecode']
     typecode   = 'V'
 
 class InstanceRefConverter(TypeConverter):
     _immutable_fields_ = ['typecode', 'clsdecl']
-    typecode    = 'V'
+    typecode = 'V'
 
     def __init__(self, space, clsdecl):
         from pypy.module._cppyy.interp_cppyy import W_CPPClassDecl
@@ -508,14 +506,14 @@
         state = space.fromcache(ffitypes.State)
         return state.c_voidp
 
-    def convert_argument(self, space, w_obj, address, call_local):
+    def convert_argument(self, space, w_obj, address):
         x = rffi.cast(rffi.VOIDPP, address)
         x[0] = rffi.cast(rffi.VOIDP, self._unwrap_object(space, w_obj))
         address = rffi.cast(capi.C_OBJECT, address)
         ba = rffi.cast(rffi.CCHARP, address)
         ba[capi.c_function_arg_typeoffset(space)] = self.typecode
 
-    def convert_argument_libffi(self, space, w_obj, address, call_local):
+    def convert_argument_libffi(self, space, w_obj, address, scratch):
         x = rffi.cast(rffi.VOIDPP, address)
         x[0] = rffi.cast(rffi.VOIDP, self._unwrap_object(space, w_obj))
 
@@ -539,7 +537,7 @@
 
 class InstanceConverter(InstanceRefConverter):
 
-    def convert_argument_libffi(self, space, w_obj, address, call_local):
+    def convert_argument_libffi(self, space, w_obj, address, scratch):
         from pypy.module._cppyy.interp_cppyy import FastCallNotPossible
         raise FastCallNotPossible       # TODO: by-value is a jit_libffi special case
 
@@ -551,9 +549,8 @@
     def to_memory(self, space, w_obj, w_value, offset):
         self._is_abstract(space)
 
-
 class InstancePtrConverter(InstanceRefConverter):
-    typecode    = 'o'
+    typecode = 'o'
 
     def _unwrap_object(self, space, w_obj):
         try:
@@ -574,36 +571,41 @@
         address[0] = rffi.cast(rffi.VOIDP, self._unwrap_object(space, w_value))
 
 class InstancePtrPtrConverter(InstancePtrConverter):
-    _immutable_fields_ = ['uses_local']
+    typecode = 'o'
 
-    uses_local = True
+    def __init__(self, space, extra):
+        InstancePtrConverter.__init__(self, space, extra)
+        self.ref_buffer = lltype.nullptr(rffi.VOIDPP.TO)
 
-    def convert_argument(self, space, w_obj, address, call_local):
-        r = rffi.cast(rffi.VOIDPP, call_local)
-        r[0] = rffi.cast(rffi.VOIDP, self._unwrap_object(space, w_obj))
+    def convert_argument(self, space, w_obj, address):
         x = rffi.cast(rffi.VOIDPP, address)
-        x[0] = rffi.cast(rffi.VOIDP, call_local)
-        address = rffi.cast(capi.C_OBJECT, address)
+        ptr = rffi.cast(rffi.VOIDP, self._unwrap_object(space, w_obj))
+        self.ref_buffer = lltype.malloc(rffi.VOIDPP.TO, 1, flavor='raw')
+        self.ref_buffer[0] = ptr
+        x[0] = self.ref_buffer
         ba = rffi.cast(rffi.CCHARP, address)
-        ba[capi.c_function_arg_typeoffset(space)] = 'o'
+        ba[capi.c_function_arg_typeoffset(space)] = self.typecode
 
-    def convert_argument_libffi(self, space, w_obj, address, call_local):
+    def convert_argument_libffi(self, space, w_obj, address, scratch):
         # TODO: finalize_call not yet called for fast call (see interp_cppyy.py)
         from pypy.module._cppyy.interp_cppyy import FastCallNotPossible
         raise FastCallNotPossible
 
-    def finalize_call(self, space, w_obj, call_local):
-        from pypy.module._cppyy.interp_cppyy import W_CPPInstance
-        assert isinstance(w_obj, W_CPPInstance)
-        r = rffi.cast(rffi.VOIDPP, call_local)
-        w_obj._rawobject = rffi.cast(capi.C_OBJECT, r[0])
-
     def from_memory(self, space, w_obj, w_pycppclass, offset):
         address = rffi.cast(capi.C_OBJECT, self._get_raw_address(space, w_obj, offset))
         from pypy.module._cppyy import interp_cppyy
         return interp_cppyy.wrap_cppinstance(
             space, address, self.clsdecl, do_cast=False, is_ref=True)
 
+    def finalize_call(self, space, w_obj):
+        if self.ref_buffer:
+            set_rawobject(space, w_obj, self.ref_buffer[0])
+
+    def free_argument(self, space, arg):
+        if self.ref_buffer:
+            lltype.free(self.ref_buffer, flavor='raw')
+            self.ref_buffer = lltype.nullptr(rffi.VOIDPP.TO)
+
 class StdStringConverter(InstanceConverter):
 
     def __init__(self, space, extra):
@@ -628,7 +630,7 @@
         except Exception:
             InstanceConverter.to_memory(self, space, w_obj, w_value, offset)
 
-    def free_argument(self, space, arg, call_local):
+    def free_argument(self, space, arg):
         capi.c_destruct(space, self.clsdecl, rffi.cast(capi.C_OBJECT, rffi.cast(rffi.VOIDPP, arg)[0]))
 
 class StdStringRefConverter(InstancePtrConverter):
@@ -646,7 +648,7 @@
         state = space.fromcache(ffitypes.State)
         return state.c_voidp
 
-    def convert_argument(self, space, w_obj, address, call_local):
+    def convert_argument(self, space, w_obj, address):
         if hasattr(space, "fake"):
             raise NotImplementedError
         space.getbuiltinmodule("cpyext")
@@ -657,7 +659,7 @@
         ba = rffi.cast(rffi.CCHARP, address)
         ba[capi.c_function_arg_typeoffset(space)] = 'a'
 
-    def convert_argument_libffi(self, space, w_obj, address, call_local):
+    def convert_argument_libffi(self, space, w_obj, address, scratch):
         # TODO: free_argument not yet called for fast call (see interp_cppyy.py)
         from pypy.module._cppyy.interp_cppyy import FastCallNotPossible
         raise FastCallNotPossible
@@ -671,7 +673,7 @@
         x = rffi.cast(rffi.VOIDPP, address)
         x[0] = rffi.cast(rffi.VOIDP, ref)"""
 
-    def free_argument(self, space, arg, call_local):
+    def free_argument(self, space, arg):
         if hasattr(space, "fake"):
             raise NotImplementedError
         space.getbuiltinmodule("cpyext")
@@ -685,7 +687,7 @@
     def __init__(self, space, signature):
         self.signature = signature
 
-    def convert_argument(self, space, w_obj, address, call_local):
+    def convert_argument(self, space, w_obj, address):
         # TODO: atm, does not actually get an overload, but a staticmethod
         from pypy.module._cppyy.interp_cppyy import W_CPPOverload
         cppol = space.interp_w(W_CPPOverload, w_obj)
@@ -740,7 +742,7 @@
         raise oefmt(space.w_TypeError,
                     "cannot pass %T instance as %s", w_obj, self.rawdecl.name)
 
-    def convert_argument(self, space, w_obj, address, call_local):
+    def convert_argument(self, space, w_obj, address):
         x = rffi.cast(rffi.VOIDPP, address)
         x[0] = rffi.cast(rffi.VOIDP, self._unwrap_object(space, w_obj))
         address = rffi.cast(capi.C_OBJECT, address)
diff --git a/pypy/module/_cppyy/interp_cppyy.py b/pypy/module/_cppyy/interp_cppyy.py
--- a/pypy/module/_cppyy/interp_cppyy.py
+++ b/pypy/module/_cppyy/interp_cppyy.py
@@ -1,6 +1,9 @@
 import pypy.module._cppyy.capi as capi
 
 from pypy.interpreter.error import OperationError, oefmt
+from pypy.interpreter.function import Method
+from pypy.interpreter.argument import Arguments
+from pypy.interpreter.typedef import interp_attrproperty_w, descr_generic_ne, make_weakref_descr
 from pypy.interpreter.gateway import interp2app, unwrap_spec
 from pypy.interpreter.typedef import TypeDef, GetSetProperty, interp_attrproperty
 from pypy.interpreter.baseobjspace import W_Root
@@ -166,11 +169,13 @@
 #  W_CPPConstructorOverload:      constructors
 #  W_CPPStaticOverload:           free and static functions
 #  W_CPPTemplateOverload:         templated methods
-#  W_CPPTemplateStaticOveload:    templated free and static functions
+#  W_CPPTemplateStaticOverload:   templated free and static functions
 #
 #  CPPMethod:         a single function or method (base class)
 #  CPPSetItem:        specialization for Python's __setitem__
 #
+#  MethodWithProps:   python instancemethod that forwards properties
+#
 # All methods/functions derive from CPPMethod and are collected as overload
 # candidates in user-facing overload classes. Templated methods are a two-step
 # process, where first the template is instantiated (or selected if already
@@ -183,13 +188,13 @@
     the memory_regulator."""
 
     _attrs_ = ['space', 'scope', 'cppmethod', 'arg_defs', 'args_required',
-               'converters', 'executor', '_funcaddr', 'cif_descr', 'uses_local']
+               'converters', 'executor', '_funcaddr', 'cif_descr']
     _immutable_fields_ = ['scope', 'cppmethod', 'arg_defs', 'args_required',
-                          'converters', 'executor', 'uses_local']
+               'converters', 'executor', '_funcaddr', 'cif_descr']
 
-    def __init__(self, space, declaring_scope, cppmethod, arg_defs, args_required):
+    def __init__(self, space, decl_scope, cppmethod, arg_defs, args_required):
         self.space = space
-        self.scope = declaring_scope
+        self.scope = decl_scope
         self.cppmethod = cppmethod
         self.arg_defs = arg_defs
         self.args_required = args_required
@@ -200,14 +205,6 @@
         self.executor = None
         self.cif_descr = lltype.nullptr(jit_libffi.CIF_DESCRIPTION)
         self._funcaddr = lltype.nullptr(capi.C_FUNC_PTR.TO)
-        self.uses_local = False
-
-    def _address_from_local_buffer(self, call_local, idx):
-        if not call_local:
-            return call_local
-        stride = 2*rffi.sizeof(rffi.VOIDP)
-        loc_idx = lltype.direct_ptradd(rffi.cast(rffi.CCHARP, call_local), idx*stride)
-        return rffi.cast(rffi.VOIDP, loc_idx)
 
     @jit.unroll_safe
     def call(self, cppthis, args_w, useffi):
@@ -233,61 +230,55 @@
             except Exception:
                 pass
 
-        # some calls, e.g. for ptr-ptr or reference need a local array to store data for
-        # the duration of the call
-        if self.uses_local:
-            call_local = lltype.malloc(rffi.VOIDP.TO, 2*len(args_w), flavor='raw')
-        else:
-            call_local = lltype.nullptr(rffi.VOIDP.TO)
+        # attempt to call directly through ffi chain
+        if useffi and self._funcaddr:
+            try:
+                return self.do_fast_call(cppthis, args_w)
+            except FastCallNotPossible:
+                pass      # can happen if converters or executor does not implement ffi
 
+        # ffi chain must have failed; using stub functions instead
+        args, stat = self.prepare_arguments(args_w)
         try:
-            # attempt to call directly through ffi chain
-            if useffi and self._funcaddr:
-                try:
-                    return self.do_fast_call(cppthis, args_w, call_local)
-                except FastCallNotPossible:
-                    pass      # can happen if converters or executor does not implement ffi
-
-            # ffi chain must have failed; using stub functions instead
-            args, stat = self.prepare_arguments(args_w, call_local)
-            try:
-                result = self.executor.execute(
-                    self.space, self.cppmethod, cppthis, len(args_w), args)
-                if stat[0] != rffi.cast(rffi.ULONG, 0):
-                    what = rffi.cast(rffi.CCHARP, stat[1])
-                    pywhat = rffi.charp2str(what)
-                    capi.c_free(self.space, rffi.cast(rffi.VOIDP, what))
-                    raise OperationError(self.space.w_Exception, self.space.newtext(pywhat))
-                return result
-            finally:
-                self.finalize_call(args, args_w, call_local)
+            result = self.executor.execute(
+                self.space, self.cppmethod, cppthis, len(args_w), args)
+            if stat[0] != rffi.cast(rffi.ULONG, 0):
+                what = rffi.cast(rffi.CCHARP, stat[1])
+                pywhat = rffi.charp2str(what)
+                capi.c_free(self.space, rffi.cast(rffi.VOIDP, what))
+                raise OperationError(self.space.w_Exception, self.space.newtext(pywhat))
+            return result
         finally:
-            if call_local:
-                lltype.free(call_local, flavor='raw')
+            self.finalize_call(args, args_w)
 
     @jit.unroll_safe
-    def do_fast_call(self, cppthis, args_w, call_local):
+    def do_fast_call(self, cppthis, args_w):
         if self.cif_descr == lltype.nullptr(jit_libffi.CIF_DESCRIPTION):
             raise FastCallNotPossible
         jit.promote(self)
         cif_descr = self.cif_descr
-        buffer = lltype.malloc(rffi.CCHARP.TO, cif_descr.exchange_size, flavor='raw')
+        # add extra space for const-ref support (see converter.py)
+        buffer = lltype.malloc(rffi.CCHARP.TO, cif_descr.exchange_size+len(self.arg_defs)*rffi.sizeof(rffi.DOUBLE), flavor='raw')
+        thisoff = 0
         try:
-            # this pointer
-            data = rffi.ptradd(buffer, cif_descr.exchange_args[0])
-            x = rffi.cast(rffi.LONGP, data)       # LONGP needed for test_zjit.py
-            x[0] = rffi.cast(rffi.LONG, cppthis)
+            if cppthis:
+                # this pointer
+                data = rffi.ptradd(buffer, cif_descr.exchange_args[0])
+                x = rffi.cast(rffi.LONGP, data)       # LONGP needed for test_zjit.py
+                x[0] = rffi.cast(rffi.LONG, cppthis)
+                thisoff = 1
 
-            # other arguments and defaults
-            i = len(self.arg_defs) + 1
+            # actual provided arguments
+            i = -1      # needed if all arguments are defaults
             for i in range(len(args_w)):
                 conv = self.converters[i]
-                w_arg = args_w[i]
-                data = rffi.ptradd(buffer, cif_descr.exchange_args[i+1])
-                conv.convert_argument_libffi(self.space, w_arg, data, call_local)
+                data = rffi.ptradd(buffer, cif_descr.exchange_args[i+thisoff])
+                scratch = rffi.ptradd(buffer, cif_descr.exchange_size+i*rffi.sizeof(rffi.DOUBLE))
+                conv.convert_argument_libffi(self.space, args_w[i], data, scratch)
+            # drop in defaults for the rest
             for j in range(i+1, len(self.arg_defs)):
                 conv = self.converters[j]
-                data = rffi.ptradd(buffer, cif_descr.exchange_args[j+1])
+                data = rffi.ptradd(buffer, cif_descr.exchange_args[j+thisoff])
                 conv.default_argument_libffi(self.space, data)
 
             assert self._funcaddr
@@ -346,22 +337,17 @@
         self.executor = executor.get_executor(
             self.space, capi.c_method_result_type(self.space, self.cppmethod))
 
-        for conv in self.converters:
-            if conv.uses_local:
-                self.uses_local = True
-                break
-
         # Each CPPMethod corresponds one-to-one to a C++ equivalent and cppthis
         # has been offset to the matching class. Hence, the libffi pointer is
         # uniquely defined and needs to be setup only once.
         funcaddr = capi.c_function_address(self.space, self.cppmethod)
-        if funcaddr and cppthis:      # TODO: methods only for now
+        if funcaddr:
             state = self.space.fromcache(ffitypes.State)
 
-            # argument type specification (incl. cppthis)
+            # argument type specification (incl. cppthis if applicable)
             fargs = []
             try:
-                fargs.append(state.c_voidp)
+                if cppthis: fargs.append(state.c_voidp)
                 for i, conv in enumerate(self.converters):
                     fargs.append(conv.cffi_type(self.space))
                 fresult = self.executor.cffi_type(self.space)
@@ -386,7 +372,7 @@
             self._funcaddr = funcaddr
 
     @jit.unroll_safe
-    def prepare_arguments(self, args_w, call_local):
+    def prepare_arguments(self, args_w):
         args = capi.c_allocate_function_args(self.space, len(args_w))
         stride = capi.c_function_arg_sizeof(self.space)
         for i in range(len(args_w)):
@@ -394,15 +380,13 @@
             w_arg = args_w[i]
             try:
                 arg_i = lltype.direct_ptradd(rffi.cast(rffi.CCHARP, args), i*stride)
-                loc_i = self._address_from_local_buffer(call_local, i)
-                conv.convert_argument(self.space, w_arg, rffi.cast(capi.C_OBJECT, arg_i), loc_i)
+                conv.convert_argument(self.space, w_arg, rffi.cast(capi.C_OBJECT, arg_i))
             except:
                 # fun :-(
                 for j in range(i):
                     conv = self.converters[j]
                     arg_j = lltype.direct_ptradd(rffi.cast(rffi.CCHARP, args), j*stride)
-                    loc_j = self._address_from_local_buffer(call_local, j)
-                    conv.free_argument(self.space, rffi.cast(capi.C_OBJECT, arg_j), loc_j)
+                    conv.free_argument(self.space, rffi.cast(capi.C_OBJECT, arg_j))
                 capi.c_deallocate_function_args(self.space, args)
                 raise
         stat = rffi.cast(rffi.ULONGP,
@@ -411,14 +395,13 @@
         return args, stat
 
     @jit.unroll_safe
-    def finalize_call(self, args, args_w, call_local):
+    def finalize_call(self, args, args_w):
         stride = capi.c_function_arg_sizeof(self.space)
         for i in range(len(args_w)):
             conv = self.converters[i]
             arg_i = lltype.direct_ptradd(rffi.cast(rffi.CCHARP, args), i*stride)
-            loc_i = self._address_from_local_buffer(call_local, i)
-            conv.finalize_call(self.space, args_w[i], loc_i)
-            conv.free_argument(self.space, rffi.cast(capi.C_OBJECT, arg_i), loc_i)
+            conv.finalize_call(self.space, args_w[i])
+            conv.free_argument(self.space, rffi.cast(capi.C_OBJECT, arg_i))
         capi.c_deallocate_function_args(self.space, args)
 
     def signature(self, show_formalargs=True):
@@ -466,42 +449,81 @@
         CPPMethod.call(self, cppthis, args_w, useffi)
 
 
+# CPPOverloads have settable flags that control memory and ffi behavior. These flags
+# need forwarding, which the normal instancemethod does not provide, hence this
+# derived class.
+class MethodWithProps(Method):
+    # allow user to determine ffi use rules per overload
+    def fget_useffi(self, space):
+	f = space.interp_w(W_CPPOverload, self.w_function)
+        return f.fget_useffi(space)
+
+    @unwrap_spec(value=bool)
+    def fset_useffi(self, space, value):
+        f = space.interp_w(W_CPPOverload, self.w_function)
+        f.fset_useffi(space, value)
+
+MethodWithProps.typedef = TypeDef(
+    "cpp_instancemethod",
+    __doc__ = """cpp_instancemethod(function, instance, class)
+
+Create an instance method object.""",
+    __new__ = interp2app(MethodWithProps.descr_method__new__.im_func),
+    __call__ = interp2app(MethodWithProps.descr_method_call),
+    __get__ = interp2app(MethodWithProps.descr_method_get),
+    im_func = interp_attrproperty_w('w_function', cls=MethodWithProps),
+    __func__ = interp_attrproperty_w('w_function', cls=MethodWithProps),
+    im_self = interp_attrproperty_w('w_instance', cls=MethodWithProps),
+    __self__ = interp_attrproperty_w('w_instance', cls=MethodWithProps),
+    im_class = interp_attrproperty_w('w_class', cls=MethodWithProps),
+    __getattribute__ = interp2app(MethodWithProps.descr_method_getattribute),
+    __eq__ = interp2app(MethodWithProps.descr_method_eq),
+    __ne__ = descr_generic_ne,
+    __hash__ = interp2app(MethodWithProps.descr_method_hash),
+    __repr__ = interp2app(MethodWithProps.descr_method_repr),
+    __reduce__ = interp2app(MethodWithProps.descr_method__reduce__),
+    __weakref__ = make_weakref_descr(MethodWithProps),
+    __useffi__ = GetSetProperty(MethodWithProps.fget_useffi, MethodWithProps.fset_useffi),
+    )
+MethodWithProps.typedef.acceptable_as_base_class = False
+
+
 class W_CPPOverload(W_Root):
     """App-level dispatcher: controls a collection of (potentially) overloaded methods
     or functions. Calls these in order and deals with error handling and reporting."""
 
-    _attrs_ = ['space', 'scope', 'functions', 'flags', 'w_this']
+    _attrs_ = ['space', 'scope', 'functions', 'flags']
     _immutable_fields_ = ['scope', 'functions[*]']
 
-    def __init__(self, space, declaring_scope, functions, flags = OVERLOAD_FLAGS_USE_FFI):
+    def __init__(self, space, decl_scope, funcs, flags = OVERLOAD_FLAGS_USE_FFI):
         self.space  = space
-        self.scope  = declaring_scope
+        self.scope  = decl_scope
         from rpython.rlib import debug
-        self.functions = debug.make_sure_not_resized(functions)
+        self.functions = debug.make_sure_not_resized(funcs)
         self.flags  = flags
-        self.w_this = self.space.w_None
+
+    def descr_get(self, w_obj, w_cls=None):
+        """functionobject.__get__(obj[, type]) -> method"""
+        # TODO: check validity of w_cls if given
+        space = self.space
+        asking_for_bound = (space.is_none(w_cls) or
+                            not space.is_w(w_obj, space.w_None) or
+                            space.is_w(w_cls, space.type(space.w_None)))
+        if asking_for_bound:
+            return MethodWithProps(space, self, w_obj, w_cls)
+        else:
+            return MethodWithProps(space, self, None, w_cls)
 
     @unwrap_spec(args_w='args_w')
-    def descr_get(self, w_cppinstance, args_w):
-        if self.space.is_w(w_cppinstance, self.space.w_None):
-            return self  # unbound, so no new instance needed
-        cppol = W_CPPOverload(self.space, self.scope, self.functions, self.flags)
-        cppol.w_this = w_cppinstance
-        return cppol     # bound
-
-    @unwrap_spec(args_w='args_w')
-    def call(self, args_w):
-        if self.space.is_w(self.w_this, self.space.w_None) and len(args_w):
-            w_this = args_w[0]
-            args_w = args_w[1:]
-        else:
-            w_this = self.w_this
+    def call_args(self, args_w):
+        jit.promote(self)
+        w_this = args_w[0]
         cppinstance = self.space.interp_w(W_CPPInstance, w_this)
         cppinstance._nullcheck()
         if not capi.c_is_subtype(self.space, cppinstance.clsdecl, self.scope):
             raise oefmt(self.space.w_TypeError,
                 "cannot pass %T instance as %s", w_this, self.scope.name)
-        return self.call_impl(cppinstance.get_cppthis(self.scope), args_w)
+        return self.call_impl(cppinstance.get_cppthis(self.scope), args_w[1:])
 
     @jit.unroll_safe
     def call_impl(self, cppthis, args_w):
@@ -576,13 +598,17 @@
     def fget_doc(self, space):
         return self.prototype()
 
+    def getname(self, space): 
+        # for the benefit of Method/instancemethod
+        return capi.c_method_name(space, self.functions[0].cppmethod)
+
     def __repr__(self):
         return "W_CPPOverload(%s)" % [f.prototype() for f in self.functions]
 
 W_CPPOverload.typedef = TypeDef(
     'CPPOverload',
     __get__    = interp2app(W_CPPOverload.descr_get),
-    __call__   = interp2app(W_CPPOverload.call),
+    __call__   = interp2app(W_CPPOverload.call_args),
     __useffi__ = GetSetProperty(W_CPPOverload.fget_useffi, W_CPPOverload.fset_useffi),
     __doc__    = GetSetProperty(W_CPPOverload.fget_doc)
 )
@@ -593,27 +619,24 @@
 class W_CPPStaticOverload(W_CPPOverload):
     _attrs_ = []
 
-    @unwrap_spec(args_w='args_w')
-    def descr_get(self, w_cppinstance, args_w):
-        if isinstance(w_cppinstance, W_CPPInstance):
+    def descr_get(self, w_obj, w_cls=None):
+        if isinstance(w_obj, W_CPPInstance):
             # two possibilities: this is a static function called on an
             # instance and w_this must not be set, or a free function rebound
             # onto a class and w_this should be set
-            cppinstance = self.space.interp_w(W_CPPInstance, w_cppinstance)
+            cppinstance = self.space.interp_w(W_CPPInstance, w_obj)
             if cppinstance.clsdecl.handle != self.scope.handle:
-                cppol = W_CPPStaticOverload(self.space, self.scope, self.functions, self.flags)
-                cppol.w_this = w_cppinstance
-                return cppol       # bound
+	        return MethodWithProps(self.space, self, w_obj, w_cls)    # bound
         return self      # unbound
 
     @unwrap_spec(args_w='args_w')
-    def call(self, args_w):
-        if not self.space.is_w(self.w_this, self.space.w_None):
-            # free function used as bound method, put self back into args_w
-            cppinstance = self.space.interp_w(W_CPPInstance, self.w_this)
-            cppinstance._nullcheck()
-            args_w = [self.w_this] + args_w
+    def call_args(self, args_w):
+        jit.promote(self)
+        #if isinstance(args_w[0], W_CPPInstance):
+            # free function used as bound method, leave in place
         return self.call_impl(capi.C_NULL_OBJECT, args_w)
+        # free functions are implemented as methods of 'namespace' classes, remove 'instance'
+        #return self.call_impl(capi.C_NULL_OBJECT, args_w[1:])
 
     def __repr__(self):
         return "W_CPPStaticOverload(%s)" % [f.prototype() for f in self.functions]
@@ -621,7 +644,7 @@
 W_CPPStaticOverload.typedef = TypeDef(
     'CPPStaticOverload',
     __get__    = interp2app(W_CPPStaticOverload.descr_get),
-    __call__   = interp2app(W_CPPStaticOverload.call),
+    __call__   = interp2app(W_CPPStaticOverload.call_args),
     __useffi__ = GetSetProperty(W_CPPStaticOverload.fget_useffi, W_CPPStaticOverload.fset_useffi),
     __doc__    = GetSetProperty(W_CPPStaticOverload.fget_doc)
 )
@@ -630,27 +653,20 @@
 class W_CPPConstructorOverload(W_CPPOverload):
     _attrs_ = []
 
-    @unwrap_spec(args_w='args_w')
-    def descr_get(self, w_cppinstance, args_w):
-        if self.space.is_w(w_cppinstance, self.space.w_None):
-            return self  # unbound (TODO: probably useless)
-        cppol = W_CPPConstructorOverload(self.space, self.scope, self.functions, self.flags)
-        cppol.w_this = w_cppinstance
-        return cppol     # bound
+    def __init__(self, space, decl_scope, funcs, flags = OVERLOAD_FLAGS_USE_FFI):
+        W_CPPOverload.__init__(self, space, decl_scope, funcs, flags)
+        self.flags &= ~OVERLOAD_FLAGS_USE_FFI
 
     @unwrap_spec(args_w='args_w')
-    def call(self, args_w):
+    def call_args(self, args_w):
+        jit.promote(self)
         # TODO: factor out the following:
         if capi.c_is_abstract(self.space, self.scope.handle):
             raise oefmt(self.space.w_TypeError,
                         "cannot instantiate abstract class '%s'",
                         self.scope.name)
-        if self.space.is_w(self.w_this, self.space.w_None) and len(args_w):
-            cppinstance = self.space.interp_w(W_CPPInstance, args_w[0])
-            args_w = args_w[1:]
-        else:
-            cppinstance = self.space.interp_w(W_CPPInstance, self.w_this)
-        w_result = self.call_impl(rffi.cast(capi.C_OBJECT, self.scope.handle), args_w)
+        cppinstance = self.space.interp_w(W_CPPInstance, args_w[0])
+        w_result = self.call_impl(rffi.cast(capi.C_OBJECT, self.scope.handle), args_w[1:])
         newthis = rffi.cast(capi.C_OBJECT, self.space.uint_w(w_result))
         if cppinstance is not None:
             cppinstance._rawobject = newthis
@@ -662,7 +678,7 @@
 W_CPPConstructorOverload.typedef = TypeDef(
     'CPPConstructorOverload',
     __get__    = interp2app(W_CPPConstructorOverload.descr_get),
-    __call__   = interp2app(W_CPPConstructorOverload.call),
+    __call__   = interp2app(W_CPPConstructorOverload.call_args),
     __doc__    = GetSetProperty(W_CPPConstructorOverload.fget_doc)
 )
 
@@ -725,7 +741,9 @@
         # try to match with run-time instantiations
         for cppol in self.master.overloads.values():
             try:
-                return cppol.descr_get(self.w_this, []).call(args_w)
+                if not self.space.is_w(self.w_this, self.space.w_None):
+                    return self.space.call_obj_args(cppol, self.w_this, Arguments(self.space, args_w))
+                return self.space.call_args(cppol, Arguments(self.space, args_w))
             except Exception:
                 pass    # completely ignore for now; have to see whether errors become confusing
 
@@ -748,7 +766,9 @@
             except KeyError:
                 self.master.overloads[fullname] = method
 
-        return method.descr_get(self.w_this, []).call(args_w)
+        if not self.space.is_w(self.w_this, self.space.w_None):
+	    return self.space.call_obj_args(method, self.w_this, Arguments(self.space, args_w))
+        return self.space.call_args(method, Arguments(self.space, args_w))
 
     def getitem_impl(self, name, args_w):
         space = self.space
@@ -771,25 +791,25 @@
             if c_fullname != fullname:
                 self.master.overloads[c_fullname] = method
 
-        return method.descr_get(self.w_this, [])
+        return method.descr_get(self.w_this, None)
 
 
 class W_CPPTemplateOverload(W_CPPOverload, TemplateOverloadMixin):
     """App-level dispatcher to allow both lookup/instantiation of templated methods and
     dispatch among overloads between templated and non-templated method."""
 
-    _attrs_ = ['name', 'overloads', 'master']
+    _attrs_ = ['name', 'overloads', 'master', 'w_this']
     _immutable_fields_ = ['name']
 
-    def __init__(self, space, name, declaring_scope, functions, flags = OVERLOAD_FLAGS_USE_FFI):
-         W_CPPOverload.__init__(self, space, declaring_scope, functions, flags)
+    def __init__(self, space, name, decl_scope, functions, flags = OVERLOAD_FLAGS_USE_FFI):
+         W_CPPOverload.__init__(self, space, decl_scope, functions, flags)
          self.name = name
          self.overloads = {}
          self.master = self
+	 self.w_this = space.w_None
 
-    @unwrap_spec(args_w='args_w')
-    def descr_get(self, w_cppinstance, args_w):
-        # like W_CPPOverload, but returns W_CPPTemplateOverload
+    def descr_get(self, w_cppinstance, w_cls=None):
+	# TODO: don't return copy, but bind in an external object (like W_CPPOverload)
         if self.space.is_w(w_cppinstance, self.space.w_None):
             return self  # unbound, so no new instance needed
         cppol = W_CPPTemplateOverload(self.space, self.name, self.scope, self.functions, self.flags)
@@ -798,13 +818,13 @@
         return cppol     # bound
 
     @unwrap_spec(args_w='args_w')
-    def call(self, args_w):
+    def call_args(self, args_w):
         # direct call: either pick non-templated overload or attempt to deduce
         # the template instantiation from the argument types
 
         # try existing overloads or compile-time instantiations
         try:
-            return W_CPPOverload.call(self, args_w)
+            return W_CPPOverload.call_args(self, args_w)
         except Exception:
             pass
 
@@ -814,6 +834,9 @@
     def getitem(self, args_w):
         return self.getitem_impl(self.name, args_w)
 
+    def getname(self, space):
+        return self.name
+
     def __repr__(self):
         return "W_CPPTemplateOverload(%s)" % [f.prototype() for f in self.functions]
 
@@ -821,7 +844,7 @@
     'CPPTemplateOverload',
     __get__     = interp2app(W_CPPTemplateOverload.descr_get),
     __getitem__ = interp2app(W_CPPTemplateOverload.getitem),
-    __call__    = interp2app(W_CPPTemplateOverload.call),
+    __call__    = interp2app(W_CPPTemplateOverload.call_args),
     __useffi__  = GetSetProperty(W_CPPTemplateOverload.fget_useffi, W_CPPTemplateOverload.fset_useffi),
     __doc__     = GetSetProperty(W_CPPTemplateOverload.fget_doc)
 )
@@ -830,18 +853,18 @@
     """App-level dispatcher to allow both lookup/instantiation of templated methods and
     dispatch among overloads between templated and non-templated method."""
 
-    _attrs_ = ['name', 'overloads', 'master']
+    _attrs_ = ['name', 'overloads', 'master', 'w_this']
     _immutable_fields_ = ['name']
 
-    def __init__(self, space, name, declaring_scope, functions, flags = OVERLOAD_FLAGS_USE_FFI):
-         W_CPPStaticOverload.__init__(self, space, declaring_scope, functions, flags)
+    def __init__(self, space, name, decl_scope, funcs, flags = OVERLOAD_FLAGS_USE_FFI):
+         W_CPPStaticOverload.__init__(self, space, decl_scope, funcs, flags)
          self.name = name
          self.overloads = {}
          self.master = self
+	 self.w_this = space.w_None
 
-    @unwrap_spec(args_w='args_w')
-    def descr_get(self, w_cppinstance, args_w):
-        # like W_CPPStaticOverload, but returns W_CPPTemplateStaticOverload
+    def descr_get(self, w_cppinstance, w_cls=None):
+	# TODO: don't return copy, but bind in an external object (like W_CPPOverload)
         if isinstance(w_cppinstance, W_CPPInstance):
             cppinstance = self.space.interp_w(W_CPPInstance, w_cppinstance)
             if cppinstance.clsdecl.handle != self.scope.handle:
@@ -852,13 +875,13 @@
         return self      # unbound
 
     @unwrap_spec(args_w='args_w')
-    def call(self, args_w):
+    def call_args(self, args_w):
         # direct call: either pick non-templated overload or attempt to deduce
         # the template instantiation from the argument types
 
         # try existing overloads or compile-time instantiations
         try:
-            return W_CPPStaticOverload.call(self, args_w)
+            return W_CPPStaticOverload.call_args(self, args_w)
         except Exception:
             pass
 
@@ -869,6 +892,9 @@
     def getitem(self, args_w):
         return self.getitem_impl(self.name, args_w)
 
+    def getname(self, space):
+        return self.name
+
     def __repr__(self):
         return "W_CPPTemplateStaticOverload(%s)" % [f.prototype() for f in self.functions]
 
@@ -876,7 +902,7 @@
     'CPPTemplateStaticOverload',
     __get__     = interp2app(W_CPPTemplateStaticOverload.descr_get),
     __getitem__ = interp2app(W_CPPTemplateStaticOverload.getitem),
-    __call__    = interp2app(W_CPPTemplateStaticOverload.call),
+    __call__    = interp2app(W_CPPTemplateStaticOverload.call_args),
     __useffi__  = GetSetProperty(W_CPPTemplateStaticOverload.fget_useffi, W_CPPTemplateStaticOverload.fset_useffi),
     __doc__     = GetSetProperty(W_CPPTemplateStaticOverload.fget_doc)
 )
@@ -898,9 +924,9 @@
     _attrs_ = ['space', 'scope', 'converter', 'offset']
     _immutable_fields = ['scope', 'converter', 'offset']
 
-    def __init__(self, space, declaring_scope, type_name, offset):
+    def __init__(self, space, decl_scope, type_name, offset):
         self.space = space
-        self.scope = declaring_scope
+        self.scope = decl_scope
         self.converter = converter.get_converter(self.space, type_name, '')
         self.offset = offset
 
@@ -1417,7 +1443,7 @@
                     ol = W_CPPStaticOverload(self.space, nss, funcs[:])
                     # TODO: cache this operator (not done yet, as the above does not
                     # select all overloads)
-                    return ol.call([self, w_other])
+                    return ol.call_args([self, w_other])
         except OperationError as e:
             if not e.match(self.space, self.space.w_TypeError):
                 raise
diff --git a/pypy/module/_cppyy/test/test_zjit.py b/pypy/module/_cppyy/test/test_zjit.py
--- a/pypy/module/_cppyy/test/test_zjit.py
+++ b/pypy/module/_cppyy/test/test_zjit.py
@@ -205,6 +205,9 @@
     def exception_match(self, typ, sub):
         return typ is sub
 
+    def is_none(self, w_obj):
+        return w_obj is None
+
     def is_w(self, w_one, w_two):
         return w_one is w_two
 
@@ -268,6 +271,9 @@
     def call_function(self, w_func, *args_w):
         return None
 
+    def call_obj_args(self, w_callable, w_obj, args):
+        return w_callable.call_args([w_obj]+args)
+
     def _freeze_(self):
         return True
 
@@ -283,19 +289,19 @@
         def f():
             cls  = interp_cppyy.scope_byname(space, "example01")
             inst = interp_cppyy._bind_object(space, FakeInt(0), cls, True)
-            cls.get_overload("__init__").descr_get(inst, []).call([FakeInt(0)])
+            cls.get_overload("__init__").descr_get(inst, []).call_args([FakeInt(0)])
             cppmethod = cls.get_overload(method_name)
             assert isinstance(inst, interp_cppyy.W_CPPInstance)
             i = 10
             while i > 0:
                 drv.jit_merge_point(inst=inst, cppmethod=cppmethod, i=i)
-                cppmethod.descr_get(inst, []).call([FakeInt(i)])
+                cppmethod.descr_get(inst, []).call_args([FakeInt(i)])
                 i -= 1
             return 7
         f()
         space = FakeSpace()
         result = self.meta_interp(f, [], listops=True, backendopt=True, listcomp=True)
-        self.check_jitcell_token_count(0)   # same for fast and slow path??
+        self.check_jitcell_token_count(1)
         # rely on replacement of capi calls to raise exception instead (see FakeSpace.__init__)
 
     @py.test.mark.dont_track_allocations("cppmethod.cif_descr kept 'leaks'")

From pypy.commits at gmail.com  Fri Jul  6 13:00:50 2018
From: pypy.commits at gmail.com (arigo)
Date: Fri, 06 Jul 2018 10:00:50 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: hg merge default
Message-ID: <5b3fa042.1c69fb81.c496f.b6c5@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r94816:8c7fd9e901c7
Date: 2018-07-06 16:27 +0200
http://bitbucket.org/pypy/pypy/changeset/8c7fd9e901c7/

Log:	hg merge default

diff --git a/pypy/module/_cppyy/converter.py b/pypy/module/_cppyy/converter.py
--- a/pypy/module/_cppyy/converter.py
+++ b/pypy/module/_cppyy/converter.py
@@ -82,10 +82,9 @@
 
 
 class TypeConverter(object):
-    _immutable_fields_ = ['cffi_name', 'uses_local', 'name']
+    _immutable_fields_ = ['cffi_name', 'name']
 
     cffi_name  = None
-    uses_local = False
     name       = ""
 
     def __init__(self, space, extra):
@@ -108,10 +107,10 @@
         from pypy.module._cppyy.interp_cppyy import FastCallNotPossible
         raise FastCallNotPossible
 
-    def convert_argument(self, space, w_obj, address, call_local):
+    def convert_argument(self, space, w_obj, address):
         self._is_abstract(space)
 
-    def convert_argument_libffi(self, space, w_obj, address, call_local):
+    def convert_argument_libffi(self, space, w_obj, address, scratch):
         from pypy.module._cppyy.interp_cppyy import FastCallNotPossible
         raise FastCallNotPossible
 
@@ -125,10 +124,10 @@
     def to_memory(self, space, w_obj, w_value, offset):
         self._is_abstract(space)
 
-    def finalize_call(self, space, w_obj, call_local):
+    def finalize_call(self, space, w_obj):
         pass
 
-    def free_argument(self, space, arg, call_local):
+    def free_argument(self, space, arg):
         pass
 
 
@@ -172,7 +171,7 @@
         state = space.fromcache(ffitypes.State)
         return state.c_voidp
 
-    def convert_argument(self, space, w_obj, address, call_local):
+    def convert_argument(self, space, w_obj, address):
         w_tc = space.findattr(w_obj, space.newtext('typecode'))
         if w_tc is not None and space.text_w(w_tc) != self.typecode:
             raise oefmt(space.w_TypeError,
@@ -208,7 +207,7 @@
 class NumericTypeConverterMixin(object):
     _mixin_ = True
 
-    def convert_argument_libffi(self, space, w_obj, address, call_local):
+    def convert_argument_libffi(self, space, w_obj, address, scratch):
         x = rffi.cast(self.c_ptrtype, address)
         x[0] = self._unwrap_object(space, w_obj)
 
@@ -228,26 +227,23 @@
 
 class ConstRefNumericTypeConverterMixin(NumericTypeConverterMixin):
     _mixin_ = True
-    _immutable_fields_ = ['uses_local']
-
-    uses_local = True
 
     def cffi_type(self, space):
         state = space.fromcache(ffitypes.State)
         return state.c_voidp
 
-    def convert_argument_libffi(self, space, w_obj, address, call_local):
-        assert rffi.sizeof(self.c_type) <= 2*rffi.sizeof(rffi.VOIDP)  # see interp_cppyy.py
+    def convert_argument_libffi(self, space, w_obj, address, scratch):
         obj = self._unwrap_object(space, w_obj)
-        typed_buf = rffi.cast(self.c_ptrtype, call_local)
+        typed_buf = rffi.cast(self.c_ptrtype, scratch)
         typed_buf[0] = obj
         x = rffi.cast(rffi.VOIDPP, address)
-        x[0] = call_local
+        x[0] = scratch
+
 
 class IntTypeConverterMixin(NumericTypeConverterMixin):
     _mixin_ = True
 
-    def convert_argument(self, space, w_obj, address, call_local):
+    def convert_argument(self, space, w_obj, address):
         x = rffi.cast(self.c_ptrtype, address)
         x[0] = self._unwrap_object(space, w_obj)
         ba = rffi.cast(rffi.CCHARP, address)
@@ -256,7 +252,7 @@
 class FloatTypeConverterMixin(NumericTypeConverterMixin):
     _mixin_ = True
 
-    def convert_argument(self, space, w_obj, address, call_local):
+    def convert_argument(self, space, w_obj, address):
         x = rffi.cast(self.c_ptrtype, address)
         x[0] = self._unwrap_object(space, w_obj)
         ba = rffi.cast(rffi.CCHARP, address)
@@ -273,18 +269,18 @@
         state = space.fromcache(ffitypes.State)
         return state.c_void
 
-    def convert_argument(self, space, w_obj, address, call_local):
+    def convert_argument(self, space, w_obj, address):
         self._is_abstract(space)
 
 
 class BoolConverter(ffitypes.typeid(bool), TypeConverter):
-    def convert_argument(self, space, w_obj, address, call_local):
+    def convert_argument(self, space, w_obj, address):
         x = rffi.cast(rffi.LONGP, address)
         x[0] = self._unwrap_object(space, w_obj)
         ba = rffi.cast(rffi.CCHARP, address)
         ba[capi.c_function_arg_typeoffset(space)] = 'b'
 
-    def convert_argument_libffi(self, space, w_obj, address, call_local):
+    def convert_argument_libffi(self, space, w_obj, address, scratch):
         x = rffi.cast(rffi.LONGP, address)
         x[0] = self._unwrap_object(space, w_obj)
 
@@ -303,13 +299,13 @@
             address[0] = '\x00'
 
 class CharConverter(ffitypes.typeid(rffi.CHAR), TypeConverter):
-    def convert_argument(self, space, w_obj, address, call_local):
+    def convert_argument(self, space, w_obj, address):
         x = rffi.cast(rffi.CCHARP, address)
         x[0] = self._unwrap_object(space, w_obj)
         ba = rffi.cast(rffi.CCHARP, address)
         ba[capi.c_function_arg_typeoffset(space)] = 'b'
 
-    def convert_argument_libffi(self, space, w_obj, address, call_local):
+    def convert_argument_libffi(self, space, w_obj, address, scratch):
         x = rffi.cast(self.c_ptrtype, address)
         x[0] = self._unwrap_object(space, w_obj)
 
@@ -348,7 +344,7 @@
         state = space.fromcache(ffitypes.State)
         return state.c_voidp
 
-    def convert_argument_libffi(self, space, w_obj, address, call_local):
+    def convert_argument_libffi(self, space, w_obj, address, scratch):
         from pypy.module._cppyy.interp_cppyy import FastCallNotPossible
         raise FastCallNotPossible
 
@@ -381,7 +377,7 @@
 
 
 class CStringConverter(TypeConverter):
-    def convert_argument(self, space, w_obj, address, call_local):
+    def convert_argument(self, space, w_obj, address):
         x = rffi.cast(rffi.LONGP, address)
         arg = space.text_w(w_obj)
         x[0] = rffi.cast(rffi.LONG, rffi.str2charp(arg))
@@ -393,7 +389,7 @@
         charpptr = rffi.cast(rffi.CCHARPP, address)
         return space.newtext(rffi.charp2str(charpptr[0]))
 
-    def free_argument(self, space, arg, call_local):
+    def free_argument(self, space, arg):
         lltype.free(rffi.cast(rffi.CCHARPP, arg)[0], flavor='raw')
 
 class CStringConverterWithSize(CStringConverter):
@@ -423,13 +419,13 @@
         state = space.fromcache(ffitypes.State)
         return state.c_voidp
 
-    def convert_argument(self, space, w_obj, address, call_local):
+    def convert_argument(self, space, w_obj, address):
         x = rffi.cast(rffi.VOIDPP, address)
         x[0] = self._unwrap_object(space, w_obj)
         ba = rffi.cast(rffi.CCHARP, address)
         ba[capi.c_function_arg_typeoffset(space)] = 'o'
 
-    def convert_argument_libffi(self, space, w_obj, address, call_local):
+    def convert_argument_libffi(self, space, w_obj, address, scratch):
         x = rffi.cast(rffi.VOIDPP, address)
         x[0] = self._unwrap_object(space, w_obj)
 
@@ -452,37 +448,39 @@
             address[0] = rffi.cast(rffi.VOIDP, self._unwrap_object(space, w_value))
 
 class VoidPtrPtrConverter(TypeConverter):
-    _immutable_fields_ = ['uses_local', 'typecode']
+    typecode = 'p'
 
-    uses_local = True
-    typecode = 'a'
+    def __init__(self, space, extra):
+        self.ref_buffer = lltype.nullptr(rffi.VOIDPP.TO)
 
-    def convert_argument(self, space, w_obj, address, call_local):
+    def convert_argument(self, space, w_obj, address):
         x = rffi.cast(rffi.VOIDPP, address)
-        ba = rffi.cast(rffi.CCHARP, address)
         try:
             x[0] = get_rawbuffer(space, w_obj)
         except TypeError:
-            r = rffi.cast(rffi.VOIDPP, call_local)
-            r[0] = rffi.cast(rffi.VOIDP, get_rawobject(space, w_obj))
-            x[0] = rffi.cast(rffi.VOIDP, call_local)
+            ptr = rffi.cast(rffi.VOIDP, get_rawobject(space, w_obj))
+            self.ref_buffer = lltype.malloc(rffi.VOIDPP.TO, 1, flavor='raw')
+            self.ref_buffer[0] = ptr
+            x[0] = self.ref_buffer
+        ba = rffi.cast(rffi.CCHARP, address)
         ba[capi.c_function_arg_typeoffset(space)] = self.typecode
 
-    def finalize_call(self, space, w_obj, call_local):
-        r = rffi.cast(rffi.VOIDPP, call_local)
-        try:
-            set_rawobject(space, w_obj, r[0])
-        except OperationError:
-            pass             # no set on buffer/array/None
+    def finalize_call(self, space, w_obj):
+        if self.ref_buffer:
+            set_rawobject(space, w_obj, self.ref_buffer[0])
+
+    def free_argument(self, space, arg):
+        if self.ref_buffer:
+            lltype.free(self.ref_buffer, flavor='raw')
+            self.ref_buffer = lltype.nullptr(rffi.VOIDPP.TO)
 
 class VoidPtrRefConverter(VoidPtrPtrConverter):
-    _immutable_fields_ = ['uses_local', 'typecode']
-    uses_local = True
+    _immutable_fields_ = ['typecode']
     typecode   = 'V'
 
 class InstanceRefConverter(TypeConverter):
     _immutable_fields_ = ['typecode', 'clsdecl']
-    typecode    = 'V'
+    typecode = 'V'
 
     def __init__(self, space, clsdecl):
         from pypy.module._cppyy.interp_cppyy import W_CPPClassDecl
@@ -508,14 +506,14 @@
         state = space.fromcache(ffitypes.State)
         return state.c_voidp
 
-    def convert_argument(self, space, w_obj, address, call_local):
+    def convert_argument(self, space, w_obj, address):
         x = rffi.cast(rffi.VOIDPP, address)
         x[0] = rffi.cast(rffi.VOIDP, self._unwrap_object(space, w_obj))
         address = rffi.cast(capi.C_OBJECT, address)
         ba = rffi.cast(rffi.CCHARP, address)
         ba[capi.c_function_arg_typeoffset(space)] = self.typecode
 
-    def convert_argument_libffi(self, space, w_obj, address, call_local):
+    def convert_argument_libffi(self, space, w_obj, address, scratch):
         x = rffi.cast(rffi.VOIDPP, address)
         x[0] = rffi.cast(rffi.VOIDP, self._unwrap_object(space, w_obj))
 
@@ -539,7 +537,7 @@
 
 class InstanceConverter(InstanceRefConverter):
 
-    def convert_argument_libffi(self, space, w_obj, address, call_local):
+    def convert_argument_libffi(self, space, w_obj, address, scratch):
         from pypy.module._cppyy.interp_cppyy import FastCallNotPossible
         raise FastCallNotPossible       # TODO: by-value is a jit_libffi special case
 
@@ -551,9 +549,8 @@
     def to_memory(self, space, w_obj, w_value, offset):
         self._is_abstract(space)
 
-
 class InstancePtrConverter(InstanceRefConverter):
-    typecode    = 'o'
+    typecode = 'o'
 
     def _unwrap_object(self, space, w_obj):
         try:
@@ -574,36 +571,41 @@
         address[0] = rffi.cast(rffi.VOIDP, self._unwrap_object(space, w_value))
 
 class InstancePtrPtrConverter(InstancePtrConverter):
-    _immutable_fields_ = ['uses_local']
+    typecode = 'o'
 
-    uses_local = True
+    def __init__(self, space, extra):
+        InstancePtrConverter.__init__(self, space, extra)
+        self.ref_buffer = lltype.nullptr(rffi.VOIDPP.TO)
 
-    def convert_argument(self, space, w_obj, address, call_local):
-        r = rffi.cast(rffi.VOIDPP, call_local)
-        r[0] = rffi.cast(rffi.VOIDP, self._unwrap_object(space, w_obj))
+    def convert_argument(self, space, w_obj, address):
         x = rffi.cast(rffi.VOIDPP, address)
-        x[0] = rffi.cast(rffi.VOIDP, call_local)
-        address = rffi.cast(capi.C_OBJECT, address)
+        ptr = rffi.cast(rffi.VOIDP, self._unwrap_object(space, w_obj))
+        self.ref_buffer = lltype.malloc(rffi.VOIDPP.TO, 1, flavor='raw')
+        self.ref_buffer[0] = ptr
+        x[0] = self.ref_buffer
         ba = rffi.cast(rffi.CCHARP, address)
-        ba[capi.c_function_arg_typeoffset(space)] = 'o'
+        ba[capi.c_function_arg_typeoffset(space)] = self.typecode
 
-    def convert_argument_libffi(self, space, w_obj, address, call_local):
+    def convert_argument_libffi(self, space, w_obj, address, scratch):
         # TODO: finalize_call not yet called for fast call (see interp_cppyy.py)
         from pypy.module._cppyy.interp_cppyy import FastCallNotPossible
         raise FastCallNotPossible
 
-    def finalize_call(self, space, w_obj, call_local):
-        from pypy.module._cppyy.interp_cppyy import W_CPPInstance
-        assert isinstance(w_obj, W_CPPInstance)
-        r = rffi.cast(rffi.VOIDPP, call_local)
-        w_obj._rawobject = rffi.cast(capi.C_OBJECT, r[0])
-
     def from_memory(self, space, w_obj, w_pycppclass, offset):
         address = rffi.cast(capi.C_OBJECT, self._get_raw_address(space, w_obj, offset))
         from pypy.module._cppyy import interp_cppyy
         return interp_cppyy.wrap_cppinstance(
             space, address, self.clsdecl, do_cast=False, is_ref=True)
 
+    def finalize_call(self, space, w_obj):
+        if self.ref_buffer:
+            set_rawobject(space, w_obj, self.ref_buffer[0])
+
+    def free_argument(self, space, arg):
+        if self.ref_buffer:
+            lltype.free(self.ref_buffer, flavor='raw')
+            self.ref_buffer = lltype.nullptr(rffi.VOIDPP.TO)
+
 class StdStringConverter(InstanceConverter):
 
     def __init__(self, space, extra):
@@ -628,7 +630,7 @@
         except Exception:
             InstanceConverter.to_memory(self, space, w_obj, w_value, offset)
 
-    def free_argument(self, space, arg, call_local):
+    def free_argument(self, space, arg):
         capi.c_destruct(space, self.clsdecl, rffi.cast(capi.C_OBJECT, rffi.cast(rffi.VOIDPP, arg)[0]))
 
 class StdStringRefConverter(InstancePtrConverter):
@@ -646,7 +648,7 @@
         state = space.fromcache(ffitypes.State)
         return state.c_voidp
 
-    def convert_argument(self, space, w_obj, address, call_local):
+    def convert_argument(self, space, w_obj, address):
         if hasattr(space, "fake"):
             raise NotImplementedError
         space.getbuiltinmodule("cpyext")
@@ -657,7 +659,7 @@
         ba = rffi.cast(rffi.CCHARP, address)
         ba[capi.c_function_arg_typeoffset(space)] = 'a'
 
-    def convert_argument_libffi(self, space, w_obj, address, call_local):
+    def convert_argument_libffi(self, space, w_obj, address, scratch):
         # TODO: free_argument not yet called for fast call (see interp_cppyy.py)
         from pypy.module._cppyy.interp_cppyy import FastCallNotPossible
         raise FastCallNotPossible
@@ -671,7 +673,7 @@
         x = rffi.cast(rffi.VOIDPP, address)
         x[0] = rffi.cast(rffi.VOIDP, ref)"""
 
-    def free_argument(self, space, arg, call_local):
+    def free_argument(self, space, arg):
         if hasattr(space, "fake"):
             raise NotImplementedError
         space.getbuiltinmodule("cpyext")
@@ -685,7 +687,7 @@
     def __init__(self, space, signature):
         self.signature = signature
 
-    def convert_argument(self, space, w_obj, address, call_local):
+    def convert_argument(self, space, w_obj, address):
         # TODO: atm, does not actually get an overload, but a staticmethod
         from pypy.module._cppyy.interp_cppyy import W_CPPOverload
         cppol = space.interp_w(W_CPPOverload, w_obj)
@@ -740,7 +742,7 @@
         raise oefmt(space.w_TypeError,
                     "cannot pass %T instance as %s", w_obj, self.rawdecl.name)
 
-    def convert_argument(self, space, w_obj, address, call_local):
+    def convert_argument(self, space, w_obj, address):
         x = rffi.cast(rffi.VOIDPP, address)
         x[0] = rffi.cast(rffi.VOIDP, self._unwrap_object(space, w_obj))
         address = rffi.cast(capi.C_OBJECT, address)
diff --git a/pypy/module/_cppyy/interp_cppyy.py b/pypy/module/_cppyy/interp_cppyy.py
--- a/pypy/module/_cppyy/interp_cppyy.py
+++ b/pypy/module/_cppyy/interp_cppyy.py
@@ -1,6 +1,9 @@
 import pypy.module._cppyy.capi as capi
 
 from pypy.interpreter.error import OperationError, oefmt
+from pypy.interpreter.function import Method
+from pypy.interpreter.argument import Arguments
+from pypy.interpreter.typedef import interp_attrproperty_w, descr_generic_ne, make_weakref_descr
 from pypy.interpreter.gateway import interp2app, unwrap_spec
 from pypy.interpreter.typedef import TypeDef, GetSetProperty, interp_attrproperty
 from pypy.interpreter.baseobjspace import W_Root
@@ -166,11 +169,13 @@
 #  W_CPPConstructorOverload:      constructors
 #  W_CPPStaticOverload:           free and static functions
 #  W_CPPTemplateOverload:         templated methods
-#  W_CPPTemplateStaticOveload:    templated free and static functions
+#  W_CPPTemplateStaticOverload:   templated free and static functions
 #
 #  CPPMethod:         a single function or method (base class)
 #  CPPSetItem:        specialization for Python's __setitem__
 #
+#  MethodWithProps:   python instancemethod that forwards properties
+#
 # All methods/functions derive from CPPMethod and are collected as overload
 # candidates in user-facing overload classes. Templated methods are a two-step
 # process, where first the template is instantiated (or selected if already
@@ -183,13 +188,13 @@
     the memory_regulator."""
 
     _attrs_ = ['space', 'scope', 'cppmethod', 'arg_defs', 'args_required',
-               'converters', 'executor', '_funcaddr', 'cif_descr', 'uses_local']
+               'converters', 'executor', '_funcaddr', 'cif_descr']
     _immutable_fields_ = ['scope', 'cppmethod', 'arg_defs', 'args_required',
-                          'converters', 'executor', 'uses_local']
+               'converters', 'executor', '_funcaddr', 'cif_descr']
 
-    def __init__(self, space, declaring_scope, cppmethod, arg_defs, args_required):
+    def __init__(self, space, decl_scope, cppmethod, arg_defs, args_required):
         self.space = space
-        self.scope = declaring_scope
+        self.scope = decl_scope
         self.cppmethod = cppmethod
         self.arg_defs = arg_defs
         self.args_required = args_required
@@ -200,14 +205,6 @@
         self.executor = None
         self.cif_descr = lltype.nullptr(jit_libffi.CIF_DESCRIPTION)
         self._funcaddr = lltype.nullptr(capi.C_FUNC_PTR.TO)
-        self.uses_local = False
-
-    def _address_from_local_buffer(self, call_local, idx):
-        if not call_local:
-            return call_local
-        stride = 2*rffi.sizeof(rffi.VOIDP)
-        loc_idx = lltype.direct_ptradd(rffi.cast(rffi.CCHARP, call_local), idx*stride)
-        return rffi.cast(rffi.VOIDP, loc_idx)
 
     @jit.unroll_safe
     def call(self, cppthis, args_w, useffi):
@@ -233,61 +230,55 @@
             except Exception:
                 pass
 
-        # some calls, e.g. for ptr-ptr or reference need a local array to store data for
-        # the duration of the call
-        if self.uses_local:
-            call_local = lltype.malloc(rffi.VOIDP.TO, 2*len(args_w), flavor='raw')
-        else:
-            call_local = lltype.nullptr(rffi.VOIDP.TO)
+        # attempt to call directly through ffi chain
+        if useffi and self._funcaddr:
+            try:
+                return self.do_fast_call(cppthis, args_w)
+            except FastCallNotPossible:
+                pass      # can happen if converters or executor does not implement ffi
 
+        # ffi chain must have failed; using stub functions instead
+        args, stat = self.prepare_arguments(args_w)
         try:
-            # attempt to call directly through ffi chain
-            if useffi and self._funcaddr:
-                try:
-                    return self.do_fast_call(cppthis, args_w, call_local)
-                except FastCallNotPossible:
-                    pass      # can happen if converters or executor does not implement ffi
-
-            # ffi chain must have failed; using stub functions instead
-            args, stat = self.prepare_arguments(args_w, call_local)
-            try:
-                result = self.executor.execute(
-                    self.space, self.cppmethod, cppthis, len(args_w), args)
-                if stat[0] != rffi.cast(rffi.ULONG, 0):
-                    what = rffi.cast(rffi.CCHARP, stat[1])
-                    pywhat = rffi.charp2str(what)
-                    capi.c_free(self.space, rffi.cast(rffi.VOIDP, what))
-                    raise OperationError(self.space.w_Exception, self.space.newtext(pywhat))
-                return result
-            finally:
-                self.finalize_call(args, args_w, call_local)
+            result = self.executor.execute(
+                self.space, self.cppmethod, cppthis, len(args_w), args)
+            if stat[0] != rffi.cast(rffi.ULONG, 0):
+                what = rffi.cast(rffi.CCHARP, stat[1])
+                pywhat = rffi.charp2str(what)
+                capi.c_free(self.space, rffi.cast(rffi.VOIDP, what))
+                raise OperationError(self.space.w_Exception, self.space.newtext(pywhat))
+            return result
         finally:
-            if call_local:
-                lltype.free(call_local, flavor='raw')
+            self.finalize_call(args, args_w)
 
     @jit.unroll_safe
-    def do_fast_call(self, cppthis, args_w, call_local):
+    def do_fast_call(self, cppthis, args_w):
         if self.cif_descr == lltype.nullptr(jit_libffi.CIF_DESCRIPTION):
             raise FastCallNotPossible
         jit.promote(self)
         cif_descr = self.cif_descr
-        buffer = lltype.malloc(rffi.CCHARP.TO, cif_descr.exchange_size, flavor='raw')
+        # add extra space for const-ref support (see converter.py)
+        buffer = lltype.malloc(rffi.CCHARP.TO, cif_descr.exchange_size+len(self.arg_defs)*rffi.sizeof(rffi.DOUBLE), flavor='raw')
+        thisoff = 0
         try:
-            # this pointer
-            data = rffi.ptradd(buffer, cif_descr.exchange_args[0])
-            x = rffi.cast(rffi.LONGP, data)       # LONGP needed for test_zjit.py
-            x[0] = rffi.cast(rffi.LONG, cppthis)
+            if cppthis:
+                # this pointer
+                data = rffi.ptradd(buffer, cif_descr.exchange_args[0])
+                x = rffi.cast(rffi.LONGP, data)       # LONGP needed for test_zjit.py
+                x[0] = rffi.cast(rffi.LONG, cppthis)
+                thisoff = 1
 
-            # other arguments and defaults
-            i = len(self.arg_defs) + 1
+            # actual provided arguments
+            i = -1      # needed if all arguments are defaults
             for i in range(len(args_w)):
                 conv = self.converters[i]
-                w_arg = args_w[i]
-                data = rffi.ptradd(buffer, cif_descr.exchange_args[i+1])
-                conv.convert_argument_libffi(self.space, w_arg, data, call_local)
+                data = rffi.ptradd(buffer, cif_descr.exchange_args[i+thisoff])
+                scratch = rffi.ptradd(buffer, cif_descr.exchange_size+i*rffi.sizeof(rffi.DOUBLE))
+                conv.convert_argument_libffi(self.space, args_w[i], data, scratch)
+            # drop in defaults for the rest
             for j in range(i+1, len(self.arg_defs)):
                 conv = self.converters[j]
-                data = rffi.ptradd(buffer, cif_descr.exchange_args[j+1])
+                data = rffi.ptradd(buffer, cif_descr.exchange_args[j+thisoff])
                 conv.default_argument_libffi(self.space, data)
 
             assert self._funcaddr
@@ -346,22 +337,17 @@
         self.executor = executor.get_executor(
             self.space, capi.c_method_result_type(self.space, self.cppmethod))
 
-        for conv in self.converters:
-            if conv.uses_local:
-                self.uses_local = True
-                break
-
         # Each CPPMethod corresponds one-to-one to a C++ equivalent and cppthis
         # has been offset to the matching class. Hence, the libffi pointer is
         # uniquely defined and needs to be setup only once.
         funcaddr = capi.c_function_address(self.space, self.cppmethod)
-        if funcaddr and cppthis:      # TODO: methods only for now
+        if funcaddr:
             state = self.space.fromcache(ffitypes.State)
 
-            # argument type specification (incl. cppthis)
+            # argument type specification (incl. cppthis if applicable)
             fargs = []
             try:
-                fargs.append(state.c_voidp)
+                if cppthis: fargs.append(state.c_voidp)
                 for i, conv in enumerate(self.converters):
                     fargs.append(conv.cffi_type(self.space))
                 fresult = self.executor.cffi_type(self.space)
@@ -386,7 +372,7 @@
             self._funcaddr = funcaddr
 
     @jit.unroll_safe
-    def prepare_arguments(self, args_w, call_local):
+    def prepare_arguments(self, args_w):
         args = capi.c_allocate_function_args(self.space, len(args_w))
         stride = capi.c_function_arg_sizeof(self.space)
         for i in range(len(args_w)):
@@ -394,15 +380,13 @@
             w_arg = args_w[i]
             try:
                 arg_i = lltype.direct_ptradd(rffi.cast(rffi.CCHARP, args), i*stride)
-                loc_i = self._address_from_local_buffer(call_local, i)
-                conv.convert_argument(self.space, w_arg, rffi.cast(capi.C_OBJECT, arg_i), loc_i)
+                conv.convert_argument(self.space, w_arg, rffi.cast(capi.C_OBJECT, arg_i))
             except:
                 # fun :-(
                 for j in range(i):
                     conv = self.converters[j]
                     arg_j = lltype.direct_ptradd(rffi.cast(rffi.CCHARP, args), j*stride)
-                    loc_j = self._address_from_local_buffer(call_local, j)
-                    conv.free_argument(self.space, rffi.cast(capi.C_OBJECT, arg_j), loc_j)
+                    conv.free_argument(self.space, rffi.cast(capi.C_OBJECT, arg_j))
                 capi.c_deallocate_function_args(self.space, args)
                 raise
         stat = rffi.cast(rffi.ULONGP,
@@ -411,14 +395,13 @@
         return args, stat
 
     @jit.unroll_safe
-    def finalize_call(self, args, args_w, call_local):
+    def finalize_call(self, args, args_w):
         stride = capi.c_function_arg_sizeof(self.space)
         for i in range(len(args_w)):
             conv = self.converters[i]
             arg_i = lltype.direct_ptradd(rffi.cast(rffi.CCHARP, args), i*stride)
-            loc_i = self._address_from_local_buffer(call_local, i)
-            conv.finalize_call(self.space, args_w[i], loc_i)
-            conv.free_argument(self.space, rffi.cast(capi.C_OBJECT, arg_i), loc_i)
+            conv.finalize_call(self.space, args_w[i])
+            conv.free_argument(self.space, rffi.cast(capi.C_OBJECT, arg_i))
         capi.c_deallocate_function_args(self.space, args)
 
     def signature(self, show_formalargs=True):
@@ -466,42 +449,81 @@
         CPPMethod.call(self, cppthis, args_w, useffi)
 
 
+# CPPOverloads have settable flags that control memory and ffi behavior. These flags
+# need forwarding, which the normal instancemethod does not provide, hence this
+# derived class.
+class MethodWithProps(Method):
+    # allow user to determine ffi use rules per overload
+    def fget_useffi(self, space):
+	f = space.interp_w(W_CPPOverload, self.w_function)
+        return f.fget_useffi(space)
+
+    @unwrap_spec(value=bool)
+    def fset_useffi(self, space, value):
+        f = space.interp_w(W_CPPOverload, self.w_function)
+        f.fset_useffi(space, value)
+
+MethodWithProps.typedef = TypeDef(
+    "cpp_instancemethod",
+    __doc__ = """cpp_instancemethod(function, instance, class)
+
+Create an instance method object.""",
+    __new__ = interp2app(MethodWithProps.descr_method__new__.im_func),
+    __call__ = interp2app(MethodWithProps.descr_method_call),
+    __get__ = interp2app(MethodWithProps.descr_method_get),
+    im_func = interp_attrproperty_w('w_function', cls=MethodWithProps),
+    __func__ = interp_attrproperty_w('w_function', cls=MethodWithProps),
+    im_self = interp_attrproperty_w('w_instance', cls=MethodWithProps),
+    __self__ = interp_attrproperty_w('w_instance', cls=MethodWithProps),
+    im_class = interp_attrproperty_w('w_class', cls=MethodWithProps),
+    __getattribute__ = interp2app(MethodWithProps.descr_method_getattribute),
+    __eq__ = interp2app(MethodWithProps.descr_method_eq),
+    __ne__ = descr_generic_ne,
+    __hash__ = interp2app(MethodWithProps.descr_method_hash),
+    __repr__ = interp2app(MethodWithProps.descr_method_repr),
+    __reduce__ = interp2app(MethodWithProps.descr_method__reduce__),
+    __weakref__ = make_weakref_descr(MethodWithProps),
+    __useffi__ = GetSetProperty(MethodWithProps.fget_useffi, MethodWithProps.fset_useffi),
+    )
+MethodWithProps.typedef.acceptable_as_base_class = False
+
+
 class W_CPPOverload(W_Root):
     """App-level dispatcher: controls a collection of (potentially) overloaded methods
     or functions. Calls these in order and deals with error handling and reporting."""
 
-    _attrs_ = ['space', 'scope', 'functions', 'flags', 'w_this']
+    _attrs_ = ['space', 'scope', 'functions', 'flags']
     _immutable_fields_ = ['scope', 'functions[*]']
 
-    def __init__(self, space, declaring_scope, functions, flags = OVERLOAD_FLAGS_USE_FFI):
+    def __init__(self, space, decl_scope, funcs, flags = OVERLOAD_FLAGS_USE_FFI):
         self.space  = space
-        self.scope  = declaring_scope
+        self.scope  = decl_scope
         from rpython.rlib import debug
-        self.functions = debug.make_sure_not_resized(functions)
+        self.functions = debug.make_sure_not_resized(funcs)
         self.flags  = flags
-        self.w_this = self.space.w_None
+
+    def descr_get(self, w_obj, w_cls=None):
+        """functionobject.__get__(obj[, type]) -> method"""
+        # TODO: check validity of w_cls if given
+        space = self.space
+        asking_for_bound = (space.is_none(w_cls) or
+                            not space.is_w(w_obj, space.w_None) or
+                            space.is_w(w_cls, space.type(space.w_None)))
+        if asking_for_bound:
+            return MethodWithProps(space, self, w_obj, w_cls)
+        else:
+            return MethodWithProps(space, self, None, w_cls)
 
     @unwrap_spec(args_w='args_w')
-    def descr_get(self, w_cppinstance, args_w):
-        if self.space.is_w(w_cppinstance, self.space.w_None):
-            return self  # unbound, so no new instance needed
-        cppol = W_CPPOverload(self.space, self.scope, self.functions, self.flags)
-        cppol.w_this = w_cppinstance
-        return cppol     # bound
-
-    @unwrap_spec(args_w='args_w')
-    def call(self, args_w):
-        if self.space.is_w(self.w_this, self.space.w_None) and len(args_w):
-            w_this = args_w[0]
-            args_w = args_w[1:]
-        else:
-            w_this = self.w_this
+    def call_args(self, args_w):
+        jit.promote(self)
+        w_this = args_w[0]
         cppinstance = self.space.interp_w(W_CPPInstance, w_this)
         cppinstance._nullcheck()
         if not capi.c_is_subtype(self.space, cppinstance.clsdecl, self.scope):
             raise oefmt(self.space.w_TypeError,
                 "cannot pass %T instance as %s", w_this, self.scope.name)
-        return self.call_impl(cppinstance.get_cppthis(self.scope), args_w)
+        return self.call_impl(cppinstance.get_cppthis(self.scope), args_w[1:])
 
     @jit.unroll_safe
     def call_impl(self, cppthis, args_w):
@@ -576,13 +598,17 @@
     def fget_doc(self, space):
         return self.prototype()
 
+    def getname(self, space): 
+        # for the benefit of Method/instancemethod
+        return capi.c_method_name(space, self.functions[0].cppmethod)
+
     def __repr__(self):
         return "W_CPPOverload(%s)" % [f.prototype() for f in self.functions]
 
 W_CPPOverload.typedef = TypeDef(
     'CPPOverload',
     __get__    = interp2app(W_CPPOverload.descr_get),
-    __call__   = interp2app(W_CPPOverload.call),
+    __call__   = interp2app(W_CPPOverload.call_args),
     __useffi__ = GetSetProperty(W_CPPOverload.fget_useffi, W_CPPOverload.fset_useffi),
     __doc__    = GetSetProperty(W_CPPOverload.fget_doc)
 )
@@ -593,27 +619,24 @@
 class W_CPPStaticOverload(W_CPPOverload):
     _attrs_ = []
 
-    @unwrap_spec(args_w='args_w')
-    def descr_get(self, w_cppinstance, args_w):
-        if isinstance(w_cppinstance, W_CPPInstance):
+    def descr_get(self, w_obj, w_cls=None):
+        if isinstance(w_obj, W_CPPInstance):
             # two possibilities: this is a static function called on an
             # instance and w_this must not be set, or a free function rebound
             # onto a class and w_this should be set
-            cppinstance = self.space.interp_w(W_CPPInstance, w_cppinstance)
+            cppinstance = self.space.interp_w(W_CPPInstance, w_obj)
             if cppinstance.clsdecl.handle != self.scope.handle:
-                cppol = W_CPPStaticOverload(self.space, self.scope, self.functions, self.flags)
-                cppol.w_this = w_cppinstance
-                return cppol       # bound
+	        return MethodWithProps(self.space, self, w_obj, w_cls)    # bound
         return self      # unbound
 
     @unwrap_spec(args_w='args_w')
-    def call(self, args_w):
-        if not self.space.is_w(self.w_this, self.space.w_None):
-            # free function used as bound method, put self back into args_w
-            cppinstance = self.space.interp_w(W_CPPInstance, self.w_this)
-            cppinstance._nullcheck()
-            args_w = [self.w_this] + args_w
+    def call_args(self, args_w):
+        jit.promote(self)
+        #if isinstance(args_w[0], W_CPPInstance):
+            # free function used as bound method, leave in place
         return self.call_impl(capi.C_NULL_OBJECT, args_w)
+        # free functions are implemented as methods of 'namespace' classes, remove 'instance'
+        #return self.call_impl(capi.C_NULL_OBJECT, args_w[1:])
 
     def __repr__(self):
         return "W_CPPStaticOverload(%s)" % [f.prototype() for f in self.functions]
@@ -621,7 +644,7 @@
 W_CPPStaticOverload.typedef = TypeDef(
     'CPPStaticOverload',
     __get__    = interp2app(W_CPPStaticOverload.descr_get),
-    __call__   = interp2app(W_CPPStaticOverload.call),
+    __call__   = interp2app(W_CPPStaticOverload.call_args),
     __useffi__ = GetSetProperty(W_CPPStaticOverload.fget_useffi, W_CPPStaticOverload.fset_useffi),
     __doc__    = GetSetProperty(W_CPPStaticOverload.fget_doc)
 )
@@ -630,27 +653,20 @@
 class W_CPPConstructorOverload(W_CPPOverload):
     _attrs_ = []
 
-    @unwrap_spec(args_w='args_w')
-    def descr_get(self, w_cppinstance, args_w):
-        if self.space.is_w(w_cppinstance, self.space.w_None):
-            return self  # unbound (TODO: probably useless)
-        cppol = W_CPPConstructorOverload(self.space, self.scope, self.functions, self.flags)
-        cppol.w_this = w_cppinstance
-        return cppol     # bound
+    def __init__(self, space, decl_scope, funcs, flags = OVERLOAD_FLAGS_USE_FFI):
+        W_CPPOverload.__init__(self, space, decl_scope, funcs, flags)
+        self.flags &= ~OVERLOAD_FLAGS_USE_FFI
 
     @unwrap_spec(args_w='args_w')
-    def call(self, args_w):
+    def call_args(self, args_w):
+        jit.promote(self)
         # TODO: factor out the following:
         if capi.c_is_abstract(self.space, self.scope.handle):
             raise oefmt(self.space.w_TypeError,
                         "cannot instantiate abstract class '%s'",
                         self.scope.name)
-        if self.space.is_w(self.w_this, self.space.w_None) and len(args_w):
-            cppinstance = self.space.interp_w(W_CPPInstance, args_w[0])
-            args_w = args_w[1:]
-        else:
-            cppinstance = self.space.interp_w(W_CPPInstance, self.w_this)
-        w_result = self.call_impl(rffi.cast(capi.C_OBJECT, self.scope.handle), args_w)
+        cppinstance = self.space.interp_w(W_CPPInstance, args_w[0])
+        w_result = self.call_impl(rffi.cast(capi.C_OBJECT, self.scope.handle), args_w[1:])
         newthis = rffi.cast(capi.C_OBJECT, self.space.uint_w(w_result))
         if cppinstance is not None:
             cppinstance._rawobject = newthis
@@ -662,7 +678,7 @@
 W_CPPConstructorOverload.typedef = TypeDef(
     'CPPConstructorOverload',
     __get__    = interp2app(W_CPPConstructorOverload.descr_get),
-    __call__   = interp2app(W_CPPConstructorOverload.call),
+    __call__   = interp2app(W_CPPConstructorOverload.call_args),
     __doc__    = GetSetProperty(W_CPPConstructorOverload.fget_doc)
 )
 
@@ -725,7 +741,9 @@
         # try to match with run-time instantiations
         for cppol in self.master.overloads.values():
             try:
-                return cppol.descr_get(self.w_this, []).call(args_w)
+                if not self.space.is_w(self.w_this, self.space.w_None):
+                    return self.space.call_obj_args(cppol, self.w_this, Arguments(self.space, args_w))
+                return self.space.call_args(cppol, Arguments(self.space, args_w))
             except Exception:
                 pass    # completely ignore for now; have to see whether errors become confusing
 
@@ -748,7 +766,9 @@
             except KeyError:
                 self.master.overloads[fullname] = method
 
-        return method.descr_get(self.w_this, []).call(args_w)
+        if not self.space.is_w(self.w_this, self.space.w_None):
+	    return self.space.call_obj_args(method, self.w_this, Arguments(self.space, args_w))
+        return self.space.call_args(method, Arguments(self.space, args_w))
 
     def getitem_impl(self, name, args_w):
         space = self.space
@@ -771,25 +791,25 @@
             if c_fullname != fullname:
                 self.master.overloads[c_fullname] = method
 
-        return method.descr_get(self.w_this, [])
+        return method.descr_get(self.w_this, None)
 
 
 class W_CPPTemplateOverload(W_CPPOverload, TemplateOverloadMixin):
     """App-level dispatcher to allow both lookup/instantiation of templated methods and
     dispatch among overloads between templated and non-templated method."""
 
-    _attrs_ = ['name', 'overloads', 'master']
+    _attrs_ = ['name', 'overloads', 'master', 'w_this']
     _immutable_fields_ = ['name']
 
-    def __init__(self, space, name, declaring_scope, functions, flags = OVERLOAD_FLAGS_USE_FFI):
-         W_CPPOverload.__init__(self, space, declaring_scope, functions, flags)
+    def __init__(self, space, name, decl_scope, functions, flags = OVERLOAD_FLAGS_USE_FFI):
+         W_CPPOverload.__init__(self, space, decl_scope, functions, flags)
          self.name = name
          self.overloads = {}
          self.master = self
+	 self.w_this = space.w_None
 
-    @unwrap_spec(args_w='args_w')
-    def descr_get(self, w_cppinstance, args_w):
-        # like W_CPPOverload, but returns W_CPPTemplateOverload
+    def descr_get(self, w_cppinstance, w_cls=None):
+	# TODO: don't return copy, but bind in an external object (like W_CPPOverload)
         if self.space.is_w(w_cppinstance, self.space.w_None):
             return self  # unbound, so no new instance needed
         cppol = W_CPPTemplateOverload(self.space, self.name, self.scope, self.functions, self.flags)
@@ -798,13 +818,13 @@
         return cppol     # bound
 
     @unwrap_spec(args_w='args_w')
-    def call(self, args_w):
+    def call_args(self, args_w):
         # direct call: either pick non-templated overload or attempt to deduce
         # the template instantiation from the argument types
 
         # try existing overloads or compile-time instantiations
         try:
-            return W_CPPOverload.call(self, args_w)
+            return W_CPPOverload.call_args(self, args_w)
         except Exception:
             pass
 
@@ -814,6 +834,9 @@
     def getitem(self, args_w):
         return self.getitem_impl(self.name, args_w)
 
+    def getname(self, space):
+        return self.name
+
     def __repr__(self):
         return "W_CPPTemplateOverload(%s)" % [f.prototype() for f in self.functions]
 
@@ -821,7 +844,7 @@
     'CPPTemplateOverload',
     __get__     = interp2app(W_CPPTemplateOverload.descr_get),
     __getitem__ = interp2app(W_CPPTemplateOverload.getitem),
-    __call__    = interp2app(W_CPPTemplateOverload.call),
+    __call__    = interp2app(W_CPPTemplateOverload.call_args),
     __useffi__  = GetSetProperty(W_CPPTemplateOverload.fget_useffi, W_CPPTemplateOverload.fset_useffi),
     __doc__     = GetSetProperty(W_CPPTemplateOverload.fget_doc)
 )
@@ -830,18 +853,18 @@
     """App-level dispatcher to allow both lookup/instantiation of templated methods and
     dispatch among overloads between templated and non-templated method."""
 
-    _attrs_ = ['name', 'overloads', 'master']
+    _attrs_ = ['name', 'overloads', 'master', 'w_this']
     _immutable_fields_ = ['name']
 
-    def __init__(self, space, name, declaring_scope, functions, flags = OVERLOAD_FLAGS_USE_FFI):
-         W_CPPStaticOverload.__init__(self, space, declaring_scope, functions, flags)
+    def __init__(self, space, name, decl_scope, funcs, flags = OVERLOAD_FLAGS_USE_FFI):
+         W_CPPStaticOverload.__init__(self, space, decl_scope, funcs, flags)
          self.name = name
          self.overloads = {}
          self.master = self
+	 self.w_this = space.w_None
 
-    @unwrap_spec(args_w='args_w')
-    def descr_get(self, w_cppinstance, args_w):
-        # like W_CPPStaticOverload, but returns W_CPPTemplateStaticOverload
+    def descr_get(self, w_cppinstance, w_cls=None):
+	# TODO: don't return copy, but bind in an external object (like W_CPPOverload)
         if isinstance(w_cppinstance, W_CPPInstance):
             cppinstance = self.space.interp_w(W_CPPInstance, w_cppinstance)
             if cppinstance.clsdecl.handle != self.scope.handle:
@@ -852,13 +875,13 @@
         return self      # unbound
 
     @unwrap_spec(args_w='args_w')
-    def call(self, args_w):
+    def call_args(self, args_w):
         # direct call: either pick non-templated overload or attempt to deduce
         # the template instantiation from the argument types
 
         # try existing overloads or compile-time instantiations
         try:
-            return W_CPPStaticOverload.call(self, args_w)
+            return W_CPPStaticOverload.call_args(self, args_w)
         except Exception:
             pass
 
@@ -869,6 +892,9 @@
     def getitem(self, args_w):
         return self.getitem_impl(self.name, args_w)
 
+    def getname(self, space):
+        return self.name
+
     def __repr__(self):
         return "W_CPPTemplateStaticOverload(%s)" % [f.prototype() for f in self.functions]
 
@@ -876,7 +902,7 @@
     'CPPTemplateStaticOverload',
     __get__     = interp2app(W_CPPTemplateStaticOverload.descr_get),
     __getitem__ = interp2app(W_CPPTemplateStaticOverload.getitem),
-    __call__    = interp2app(W_CPPTemplateStaticOverload.call),
+    __call__    = interp2app(W_CPPTemplateStaticOverload.call_args),
     __useffi__  = GetSetProperty(W_CPPTemplateStaticOverload.fget_useffi, W_CPPTemplateStaticOverload.fset_useffi),
     __doc__     = GetSetProperty(W_CPPTemplateStaticOverload.fget_doc)
 )
@@ -898,9 +924,9 @@
     _attrs_ = ['space', 'scope', 'converter', 'offset']
     _immutable_fields = ['scope', 'converter', 'offset']
 
-    def __init__(self, space, declaring_scope, type_name, offset):
+    def __init__(self, space, decl_scope, type_name, offset):
         self.space = space
-        self.scope = declaring_scope
+        self.scope = decl_scope
         self.converter = converter.get_converter(self.space, type_name, '')
         self.offset = offset
 
@@ -1417,7 +1443,7 @@
                     ol = W_CPPStaticOverload(self.space, nss, funcs[:])
                     # TODO: cache this operator (not done yet, as the above does not
                     # select all overloads)
-                    return ol.call([self, w_other])
+                    return ol.call_args([self, w_other])
         except OperationError as e:
             if not e.match(self.space, self.space.w_TypeError):
                 raise
diff --git a/pypy/module/_cppyy/test/test_zjit.py b/pypy/module/_cppyy/test/test_zjit.py
--- a/pypy/module/_cppyy/test/test_zjit.py
+++ b/pypy/module/_cppyy/test/test_zjit.py
@@ -205,6 +205,9 @@
     def exception_match(self, typ, sub):
         return typ is sub
 
+    def is_none(self, w_obj):
+        return w_obj is None
+
     def is_w(self, w_one, w_two):
         return w_one is w_two
 
@@ -268,6 +271,9 @@
     def call_function(self, w_func, *args_w):
         return None
 
+    def call_obj_args(self, w_callable, w_obj, args):
+        return w_callable.call_args([w_obj]+args)
+
     def _freeze_(self):
         return True
 
@@ -283,19 +289,19 @@
         def f():
             cls  = interp_cppyy.scope_byname(space, "example01")
             inst = interp_cppyy._bind_object(space, FakeInt(0), cls, True)
-            cls.get_overload("__init__").descr_get(inst, []).call([FakeInt(0)])
+            cls.get_overload("__init__").descr_get(inst, []).call_args([FakeInt(0)])
             cppmethod = cls.get_overload(method_name)
             assert isinstance(inst, interp_cppyy.W_CPPInstance)
             i = 10
             while i > 0:
                 drv.jit_merge_point(inst=inst, cppmethod=cppmethod, i=i)
-                cppmethod.descr_get(inst, []).call([FakeInt(i)])
+                cppmethod.descr_get(inst, []).call_args([FakeInt(i)])
                 i -= 1
             return 7
         f()
         space = FakeSpace()
         result = self.meta_interp(f, [], listops=True, backendopt=True, listcomp=True)
-        self.check_jitcell_token_count(0)   # same for fast and slow path??
+        self.check_jitcell_token_count(1)
         # rely on replacement of capi calls to raise exception instead (see FakeSpace.__init__)
 
     @py.test.mark.dont_track_allocations("cppmethod.cif_descr kept 'leaks'")

From pypy.commits at gmail.com  Fri Jul  6 13:00:52 2018
From: pypy.commits at gmail.com (arigo)
Date: Fri, 06 Jul 2018 10:00:52 -0700 (PDT)
Subject: [pypy-commit] pypy default: Add rarithmetic.mulmod(). Use it to
 improve the performance of
Message-ID: <5b3fa044.1c69fb81.b9b5f.e0ff@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r94817:93f86b23bc8e
Date: 2018-07-06 18:21 +0200
http://bitbucket.org/pypy/pypy/changeset/93f86b23bc8e/

Log:	Add rarithmetic.mulmod(). Use it to improve the performance of

	 pow(a, b, c)

	if c is a large integer that still fits inside a W_IntObject.

	Note that this means that pow(a, b, c) now returns an int more
	systematically, and not a long, if all arguments are ints. I hope
	this is not a problem. Previously we would return sometimes an int
	and sometimes a long in this situation, based on the internal
	algorithm, like CPython.

diff --git a/pypy/module/__pypy__/__init__.py b/pypy/module/__pypy__/__init__.py
--- a/pypy/module/__pypy__/__init__.py
+++ b/pypy/module/__pypy__/__init__.py
@@ -48,6 +48,7 @@
         'int_lshift':      'interp_intop.int_lshift',
         'int_rshift':      'interp_intop.int_rshift',
         'uint_rshift':     'interp_intop.uint_rshift',
+        'int_mulmod':      'interp_intop.int_mulmod',
     }
 
 
diff --git a/pypy/module/__pypy__/interp_intop.py b/pypy/module/__pypy__/interp_intop.py
--- a/pypy/module/__pypy__/interp_intop.py
+++ b/pypy/module/__pypy__/interp_intop.py
@@ -2,7 +2,7 @@
 from rpython.rtyper.lltypesystem import lltype
 from rpython.rtyper.lltypesystem.lloperation import llop
 from rpython.rlib.rarithmetic import r_uint, intmask
-from rpython.rlib.rarithmetic import int_c_div, int_c_mod
+from rpython.rlib.rarithmetic import int_c_div, int_c_mod, mulmod
 from rpython.rlib import jit
 
 
@@ -39,3 +39,7 @@
     n = r_uint(n)
     x = llop.uint_rshift(lltype.Unsigned, n, m)
     return space.newint(intmask(x))
+
+ at unwrap_spec(a=int, b=int, c=int)
+def int_mulmod(space, a, b, c):
+    return space.newint(mulmod(a, b, c))
diff --git a/pypy/module/__pypy__/test/test_intop.py b/pypy/module/__pypy__/test/test_intop.py
--- a/pypy/module/__pypy__/test/test_intop.py
+++ b/pypy/module/__pypy__/test/test_intop.py
@@ -102,3 +102,7 @@
         assert intop.uint_rshift(-1, 1) == sys.maxsize
         assert intop.uint_rshift(-1, bits-2) == 3
         assert intop.uint_rshift(-1, bits-1) == 1
+
+    def test_mulmod(self):
+        from __pypy__ import intop
+        assert intop.int_mulmod(9373891, 9832739, 2**31-1) == 1025488209
diff --git a/pypy/objspace/std/intobject.py b/pypy/objspace/std/intobject.py
--- a/pypy/objspace/std/intobject.py
+++ b/pypy/objspace/std/intobject.py
@@ -233,20 +233,23 @@
     return wrapint(space, a)
 
 
- at jit.look_inside_iff(lambda space, iv, iw, iz:
-                     jit.isconstant(iw) and jit.isconstant(iz))
 def _pow(space, iv, iw, iz):
     """Helper for pow"""
-    if iw < 0:
-        if iz != 0:
-            raise oefmt(space.w_TypeError,
-                        "pow() 2nd argument cannot be negative when 3rd "
-                        "argument specified")
+    if iz == 0:
+        return _pow_nomod(iv, iw)
+    else:
+        return _pow_mod(space, iv, iw, iz)
+
+ at jit.look_inside_iff(lambda iv, iw: jit.isconstant(iw))
+def _pow_nomod(iv, iw):
+    if iw <= 0:
+        if iw == 0:
+            return 1
         # bounce it, since it always returns float
         raise ValueError
     temp = iv
     ix = 1
-    while iw > 0:
+    while True:
         if iw & 1:
             try:
                 ix = ovfcheck(ix * temp)
@@ -259,12 +262,40 @@
             temp = ovfcheck(temp * temp) # Square the value of temp
         except OverflowError:
             raise
-        if iz:
-            # If we did a multiplication, perform a modulo
-            ix %= iz
-            temp %= iz
-    if iz:
-        ix %= iz
+    return ix
+
+ at jit.look_inside_iff(lambda space, iv, iw, iz:
+                     jit.isconstant(iw) and jit.isconstant(iz))
+def _pow_mod(space, iv, iw, iz):
+    from rpython.rlib.rarithmetic import mulmod
+
+    if iw <= 0:
+        if iw == 0:
+            return 1 % iz   # != 1, for iz == 1 or iz < 0
+        raise oefmt(space.w_TypeError,
+                    "pow() 2nd argument cannot be negative when 3rd "
+                    "argument specified")
+    if iz < 0:
+        try:
+            iz = ovfcheck(-iz)
+        except OverflowError:
+            raise
+        iz_negative = True
+    else:
+        iz_negative = False
+
+    temp = iv
+    ix = 1
+    while True:
+        if iw & 1:
+            ix = mulmod(ix, temp, iz)
+        iw >>= 1   # Shift exponent down by 1 bit
+        if iw == 0:
+            break
+        temp = mulmod(temp, temp, iz)
+
+    if iz_negative and ix > 0:
+        ix -= iz
     return ix
 
 
diff --git a/pypy/objspace/std/test/test_intobject.py b/pypy/objspace/std/test/test_intobject.py
--- a/pypy/objspace/std/test/test_intobject.py
+++ b/pypy/objspace/std/test/test_intobject.py
@@ -1,8 +1,9 @@
 # encoding: utf-8
 import py
 import sys
+from pypy.interpreter.error import OperationError
 from pypy.objspace.std import intobject as iobj
-from rpython.rlib.rarithmetic import r_uint, is_valid_int
+from rpython.rlib.rarithmetic import r_uint, is_valid_int, intmask
 from rpython.rlib.rbigint import rbigint
 
 
@@ -186,6 +187,63 @@
         assert space.isinstance_w(v, space.w_long)
         assert space.bigint_w(v).eq(rbigint.fromlong(pow(10, 20)))
 
+    try:
+        from hypothesis import given, strategies, example
+    except ImportError:
+        pass
+    else:
+        @given(
+           a=strategies.integers(min_value=-sys.maxint-1, max_value=sys.maxint),
+           b=strategies.integers(min_value=-sys.maxint-1, max_value=sys.maxint),
+           c=strategies.integers(min_value=-sys.maxint-1, max_value=sys.maxint))
+        @example(0, 0, -sys.maxint-1)
+        @example(0, 1, -sys.maxint-1)
+        @example(1, 0, -sys.maxint-1)
+        def test_hypot_pow(self, a, b, c):
+            if c == 0:
+                return
+            #
+            # "pow(a, b, c)": if b < 0, should get an app-level TypeError.
+            # Otherwise, should always work except if c == -maxint-1
+            if b < 0:
+                expected = TypeError
+            elif b > 0 and c == -sys.maxint-1:
+                expected = OverflowError
+            else:
+                expected = pow(a, b, c)
+
+            try:
+                result = iobj._pow(self.space, a, b, c)
+            except OperationError as e:
+                assert ('TypeError: pow() 2nd argument cannot be negative '
+                        'when 3rd argument specified' == e.errorstr(self.space))
+                result = TypeError
+            except OverflowError:
+                result = OverflowError
+            assert result == expected
+
+        @given(
+           a=strategies.integers(min_value=-sys.maxint-1, max_value=sys.maxint),
+           b=strategies.integers(min_value=-sys.maxint-1, max_value=sys.maxint))
+        def test_hypot_pow_nomod(self, a, b):
+            # "a ** b": detect overflows and ValueErrors
+            if b < 0:
+                expected = ValueError
+            elif b > 128 and not (-1 <= a <= 1):
+                expected = OverflowError
+            else:
+                expected = a ** b
+                if expected != intmask(expected):
+                    expected = OverflowError
+
+            try:
+                result = iobj._pow(self.space, a, b, 0)
+            except ValueError:
+                result = ValueError
+            except OverflowError:
+                result = OverflowError
+            assert result == expected
+
     def test_neg(self):
         space = self.space
         x = 42
diff --git a/rpython/rlib/rarithmetic.py b/rpython/rlib/rarithmetic.py
--- a/rpython/rlib/rarithmetic.py
+++ b/rpython/rlib/rarithmetic.py
@@ -840,6 +840,31 @@
         return z
 
 
+ at specialize.memo()
+def check_support_int128():
+    from rpython.rtyper.lltypesystem import rffi
+    return hasattr(rffi, '__INT128_T')
+
+def mulmod(a, b, c):
+    """Computes (a * b) % c.
+    Assumes c > 0, and returns a nonnegative result.
+    """
+    assert c > 0
+    if LONG_BIT < LONGLONG_BIT:
+        a = r_longlong(a)
+        b = r_longlong(b)
+        return intmask((a * b) % c)
+    elif check_support_int128():
+        a = r_longlonglong(a)
+        b = r_longlonglong(b)
+        return intmask((a * b) % c)
+    else:
+        from rpython.rlib.rbigint import rbigint
+        a = rbigint.fromlong(a)
+        b = rbigint.fromlong(b)
+        return a.mul(b).int_mod(c).toint()
+
+
 # String parsing support
 # ---------------------------
 
diff --git a/rpython/rlib/rbigint.py b/rpython/rlib/rbigint.py
--- a/rpython/rlib/rbigint.py
+++ b/rpython/rlib/rbigint.py
@@ -1,6 +1,7 @@
 from rpython.rlib.rarithmetic import LONG_BIT, intmask, longlongmask, r_uint, r_ulonglong
 from rpython.rlib.rarithmetic import ovfcheck, r_longlong, widen
 from rpython.rlib.rarithmetic import most_neg_value_of_same_type
+from rpython.rlib.rarithmetic import check_support_int128
 from rpython.rlib.rstring import StringBuilder
 from rpython.rlib.debug import make_sure_not_resized, check_regular_int
 from rpython.rlib.objectmodel import we_are_translated, specialize, not_rpython
@@ -10,7 +11,7 @@
 
 import math, sys
 
-SUPPORT_INT128 = hasattr(rffi, '__INT128_T')
+SUPPORT_INT128 = check_support_int128()
 BYTEORDER = sys.byteorder
 
 # note about digit sizes:
diff --git a/rpython/rlib/test/test_rarithmetic.py b/rpython/rlib/test/test_rarithmetic.py
--- a/rpython/rlib/test/test_rarithmetic.py
+++ b/rpython/rlib/test/test_rarithmetic.py
@@ -1,5 +1,6 @@
 from rpython.rtyper.test.tool import BaseRtypingTest
 from rpython.rtyper.test.test_llinterp import interpret
+from rpython.rlib import rarithmetic
 from rpython.rlib.rarithmetic import *
 from rpython.rlib.rstring import ParseStringError, ParseStringOverflowError
 from hypothesis import given, strategies, assume
@@ -731,3 +732,17 @@
     py.test.raises(OverflowError, ovfcheck_int32_sub, 2**30, -2**30)
     assert ovfcheck_int32_mul(-2**16, 2**15) == -2**31
     py.test.raises(OverflowError, ovfcheck_int32_mul, -2**16, -2**15)
+
+ at given(strategies.integers(min_value=-sys.maxint-1, max_value=sys.maxint),
+       strategies.integers(min_value=-sys.maxint-1, max_value=sys.maxint),
+       strategies.integers(min_value=1, max_value=sys.maxint))
+def test_mulmod(a, b, c):
+    assert mulmod(a, b, c) == (a * b) % c
+    #
+    import rpython.rlib.rbigint  # import before patching check_support_int128
+    prev = rarithmetic.check_support_int128
+    try:
+        rarithmetic.check_support_int128 = lambda: False
+        assert mulmod(a, b, c) == (a * b) % c
+    finally:
+        rarithmetic.check_support_int128 = prev
diff --git a/rpython/translator/c/test/test_typed.py b/rpython/translator/c/test/test_typed.py
--- a/rpython/translator/c/test/test_typed.py
+++ b/rpython/translator/c/test/test_typed.py
@@ -962,3 +962,12 @@
         f = self.getcompiled(func, [int])
         res = f(2)
         assert res == 1     # and not 2
+
+    def test_mulmod(self):
+        from rpython.rlib.rarithmetic import mulmod
+
+        def func(a, b, c):
+            return mulmod(a, b, c)
+        f = self.getcompiled(func, [int, int, int])
+        res = f(1192871273, 1837632879, 2001286281)
+        assert res == 1573897320

From pypy.commits at gmail.com  Fri Jul  6 13:00:54 2018
From: pypy.commits at gmail.com (arigo)
Date: Fri, 06 Jul 2018 10:00:54 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: hg merge default
Message-ID: <5b3fa046.1c69fb81.f2279.c3fd@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r94818:4db575509642
Date: 2018-07-06 18:25 +0200
http://bitbucket.org/pypy/pypy/changeset/4db575509642/

Log:	hg merge default

diff --git a/pypy/module/__pypy__/__init__.py b/pypy/module/__pypy__/__init__.py
--- a/pypy/module/__pypy__/__init__.py
+++ b/pypy/module/__pypy__/__init__.py
@@ -48,6 +48,7 @@
         'int_lshift':      'interp_intop.int_lshift',
         'int_rshift':      'interp_intop.int_rshift',
         'uint_rshift':     'interp_intop.uint_rshift',
+        'int_mulmod':      'interp_intop.int_mulmod',
     }
 
 
diff --git a/pypy/module/__pypy__/interp_intop.py b/pypy/module/__pypy__/interp_intop.py
--- a/pypy/module/__pypy__/interp_intop.py
+++ b/pypy/module/__pypy__/interp_intop.py
@@ -2,7 +2,7 @@
 from rpython.rtyper.lltypesystem import lltype
 from rpython.rtyper.lltypesystem.lloperation import llop
 from rpython.rlib.rarithmetic import r_uint, intmask
-from rpython.rlib.rarithmetic import int_c_div, int_c_mod
+from rpython.rlib.rarithmetic import int_c_div, int_c_mod, mulmod
 from rpython.rlib import jit
 
 
@@ -39,3 +39,7 @@
     n = r_uint(n)
     x = llop.uint_rshift(lltype.Unsigned, n, m)
     return space.newint(intmask(x))
+
+ at unwrap_spec(a=int, b=int, c=int)
+def int_mulmod(space, a, b, c):
+    return space.newint(mulmod(a, b, c))
diff --git a/pypy/module/__pypy__/test/test_intop.py b/pypy/module/__pypy__/test/test_intop.py
--- a/pypy/module/__pypy__/test/test_intop.py
+++ b/pypy/module/__pypy__/test/test_intop.py
@@ -102,3 +102,7 @@
         assert intop.uint_rshift(-1, 1) == sys.maxsize
         assert intop.uint_rshift(-1, bits-2) == 3
         assert intop.uint_rshift(-1, bits-1) == 1
+
+    def test_mulmod(self):
+        from __pypy__ import intop
+        assert intop.int_mulmod(9373891, 9832739, 2**31-1) == 1025488209
diff --git a/pypy/objspace/std/intobject.py b/pypy/objspace/std/intobject.py
--- a/pypy/objspace/std/intobject.py
+++ b/pypy/objspace/std/intobject.py
@@ -371,20 +371,23 @@
     return wrapint(space, a)
 
 
- at jit.look_inside_iff(lambda space, iv, iw, iz:
-                     jit.isconstant(iw) and jit.isconstant(iz))
 def _pow(space, iv, iw, iz):
     """Helper for pow"""
-    if iw < 0:
-        if iz != 0:
-            raise oefmt(space.w_ValueError,
-                        "pow() 2nd argument cannot be negative when 3rd "
-                        "argument specified")
+    if iz == 0:
+        return _pow_nomod(iv, iw)
+    else:
+        return _pow_mod(space, iv, iw, iz)
+
+ at jit.look_inside_iff(lambda iv, iw: jit.isconstant(iw))
+def _pow_nomod(iv, iw):
+    if iw <= 0:
+        if iw == 0:
+            return 1
         # bounce it, since it always returns float
         raise ValueError
     temp = iv
     ix = 1
-    while iw > 0:
+    while True:
         if iw & 1:
             try:
                 ix = ovfcheck(ix * temp)
@@ -397,12 +400,40 @@
             temp = ovfcheck(temp * temp) # Square the value of temp
         except OverflowError:
             raise
-        if iz:
-            # If we did a multiplication, perform a modulo
-            ix %= iz
-            temp %= iz
-    if iz:
-        ix %= iz
+    return ix
+
+ at jit.look_inside_iff(lambda space, iv, iw, iz:
+                     jit.isconstant(iw) and jit.isconstant(iz))
+def _pow_mod(space, iv, iw, iz):
+    from rpython.rlib.rarithmetic import mulmod
+
+    if iw <= 0:
+        if iw == 0:
+            return 1 % iz   # != 1, for iz == 1 or iz < 0
+        raise oefmt(space.w_ValueError,
+                    "pow() 2nd argument cannot be negative when 3rd "
+                    "argument specified")
+    if iz < 0:
+        try:
+            iz = ovfcheck(-iz)
+        except OverflowError:
+            raise
+        iz_negative = True
+    else:
+        iz_negative = False
+
+    temp = iv
+    ix = 1
+    while True:
+        if iw & 1:
+            ix = mulmod(ix, temp, iz)
+        iw >>= 1   # Shift exponent down by 1 bit
+        if iw == 0:
+            break
+        temp = mulmod(temp, temp, iz)
+
+    if iz_negative and ix > 0:
+        ix -= iz
     return ix
 
 
diff --git a/pypy/objspace/std/test/test_intobject.py b/pypy/objspace/std/test/test_intobject.py
--- a/pypy/objspace/std/test/test_intobject.py
+++ b/pypy/objspace/std/test/test_intobject.py
@@ -1,7 +1,8 @@
 # encoding: utf-8
 import sys
+from pypy.interpreter.error import OperationError
 from pypy.objspace.std import intobject as iobj
-from rpython.rlib.rarithmetic import r_uint, is_valid_int
+from rpython.rlib.rarithmetic import r_uint, is_valid_int, intmask
 from rpython.rlib.rbigint import rbigint
 
 class TestW_IntObject:
@@ -169,6 +170,63 @@
         assert space.isinstance_w(v, space.w_int)
         assert space.bigint_w(v).eq(rbigint.fromlong(pow(10, 20)))
 
+    try:
+        from hypothesis import given, strategies, example
+    except ImportError:
+        pass
+    else:
+        @given(
+           a=strategies.integers(min_value=-sys.maxint-1, max_value=sys.maxint),
+           b=strategies.integers(min_value=-sys.maxint-1, max_value=sys.maxint),
+           c=strategies.integers(min_value=-sys.maxint-1, max_value=sys.maxint))
+        @example(0, 0, -sys.maxint-1)
+        @example(0, 1, -sys.maxint-1)
+        @example(1, 0, -sys.maxint-1)
+        def test_hypot_pow(self, a, b, c):
+            if c == 0:
+                return
+            #
+            # "pow(a, b, c)": if b < 0, should get an app-level ValueError.
+            # Otherwise, should always work except if c == -maxint-1
+            if b < 0:
+                expected = "app-level ValueError"
+            elif b > 0 and c == -sys.maxint-1:
+                expected = OverflowError
+            else:
+                expected = pow(a, b, c)
+
+            try:
+                result = iobj._pow(self.space, a, b, c)
+            except OperationError as e:
+                assert ('ValueError: pow() 2nd argument cannot be negative '
+                        'when 3rd argument specified' == e.errorstr(self.space))
+                result = "app-level ValueError"
+            except OverflowError:
+                result = OverflowError
+            assert result == expected
+
+        @given(
+           a=strategies.integers(min_value=-sys.maxint-1, max_value=sys.maxint),
+           b=strategies.integers(min_value=-sys.maxint-1, max_value=sys.maxint))
+        def test_hypot_pow_nomod(self, a, b):
+            # "a ** b": detect overflows and ValueErrors
+            if b < 0:
+                expected = ValueError
+            elif b > 128 and not (-1 <= a <= 1):
+                expected = OverflowError
+            else:
+                expected = a ** b
+                if expected != intmask(expected):
+                    expected = OverflowError
+
+            try:
+                result = iobj._pow(self.space, a, b, 0)
+            except ValueError:
+                result = ValueError
+            except OverflowError:
+                result = OverflowError
+            assert result == expected
+
     def test_neg(self):
         space = self.space
         x = 42
diff --git a/rpython/rlib/rarithmetic.py b/rpython/rlib/rarithmetic.py
--- a/rpython/rlib/rarithmetic.py
+++ b/rpython/rlib/rarithmetic.py
@@ -840,6 +840,31 @@
         return z
 
 
+ at specialize.memo()
+def check_support_int128():
+    from rpython.rtyper.lltypesystem import rffi
+    return hasattr(rffi, '__INT128_T')
+
+def mulmod(a, b, c):
+    """Computes (a * b) % c.
+    Assumes c > 0, and returns a nonnegative result.
+    """
+    assert c > 0
+    if LONG_BIT < LONGLONG_BIT:
+        a = r_longlong(a)
+        b = r_longlong(b)
+        return intmask((a * b) % c)
+    elif check_support_int128():
+        a = r_longlonglong(a)
+        b = r_longlonglong(b)
+        return intmask((a * b) % c)
+    else:
+        from rpython.rlib.rbigint import rbigint
+        a = rbigint.fromlong(a)
+        b = rbigint.fromlong(b)
+        return a.mul(b).int_mod(c).toint()
+
+
 # String parsing support
 # ---------------------------
 
diff --git a/rpython/rlib/rbigint.py b/rpython/rlib/rbigint.py
--- a/rpython/rlib/rbigint.py
+++ b/rpython/rlib/rbigint.py
@@ -1,6 +1,7 @@
 from rpython.rlib.rarithmetic import LONG_BIT, intmask, longlongmask, r_uint, r_ulonglong
 from rpython.rlib.rarithmetic import ovfcheck, r_longlong, widen
 from rpython.rlib.rarithmetic import most_neg_value_of_same_type
+from rpython.rlib.rarithmetic import check_support_int128
 from rpython.rlib.rstring import StringBuilder
 from rpython.rlib.debug import make_sure_not_resized, check_regular_int
 from rpython.rlib.objectmodel import we_are_translated, specialize, not_rpython
@@ -10,7 +11,7 @@
 
 import math, sys
 
-SUPPORT_INT128 = hasattr(rffi, '__INT128_T')
+SUPPORT_INT128 = check_support_int128()
 BYTEORDER = sys.byteorder
 
 # note about digit sizes:
diff --git a/rpython/rlib/test/test_rarithmetic.py b/rpython/rlib/test/test_rarithmetic.py
--- a/rpython/rlib/test/test_rarithmetic.py
+++ b/rpython/rlib/test/test_rarithmetic.py
@@ -1,5 +1,6 @@
 from rpython.rtyper.test.tool import BaseRtypingTest
 from rpython.rtyper.test.test_llinterp import interpret
+from rpython.rlib import rarithmetic
 from rpython.rlib.rarithmetic import *
 from rpython.rlib.rstring import ParseStringError, ParseStringOverflowError
 from hypothesis import given, strategies, assume
@@ -731,3 +732,17 @@
     py.test.raises(OverflowError, ovfcheck_int32_sub, 2**30, -2**30)
     assert ovfcheck_int32_mul(-2**16, 2**15) == -2**31
     py.test.raises(OverflowError, ovfcheck_int32_mul, -2**16, -2**15)
+
+ at given(strategies.integers(min_value=-sys.maxint-1, max_value=sys.maxint),
+       strategies.integers(min_value=-sys.maxint-1, max_value=sys.maxint),
+       strategies.integers(min_value=1, max_value=sys.maxint))
+def test_mulmod(a, b, c):
+    assert mulmod(a, b, c) == (a * b) % c
+    #
+    import rpython.rlib.rbigint  # import before patching check_support_int128
+    prev = rarithmetic.check_support_int128
+    try:
+        rarithmetic.check_support_int128 = lambda: False
+        assert mulmod(a, b, c) == (a * b) % c
+    finally:
+        rarithmetic.check_support_int128 = prev
diff --git a/rpython/translator/c/test/test_typed.py b/rpython/translator/c/test/test_typed.py
--- a/rpython/translator/c/test/test_typed.py
+++ b/rpython/translator/c/test/test_typed.py
@@ -962,3 +962,12 @@
         f = self.getcompiled(func, [int])
         res = f(2)
         assert res == 1     # and not 2
+
+    def test_mulmod(self):
+        from rpython.rlib.rarithmetic import mulmod
+
+        def func(a, b, c):
+            return mulmod(a, b, c)
+        f = self.getcompiled(func, [int, int, int])
+        res = f(1192871273, 1837632879, 2001286281)
+        assert res == 1573897320

From pypy.commits at gmail.com  Fri Jul  6 13:27:41 2018
From: pypy.commits at gmail.com (arigo)
Date: Fri, 06 Jul 2018 10:27:41 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Cache the pow() result for small values
 up to 1000. This is enough
Message-ID: <5b3fa68d.1c69fb81.6f9cf.3fb3@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r94819:06970e67d972
Date: 2018-07-06 19:26 +0200
http://bitbucket.org/pypy/pypy/changeset/06970e67d972/

Log:	Cache the pow() result for small values up to 1000. This is enough
	to recover the original pypy2 speed on #2854.

diff --git a/lib-python/3/fractions.py b/lib-python/3/fractions.py
--- a/lib-python/3/fractions.py
+++ b/lib-python/3/fractions.py
@@ -566,7 +566,7 @@
         else:
             return Fraction(round(self / shift) * shift)
 
-    def __hash__(self):
+    def __hash__(self, CACHE=[-1] * 1001):
         """hash(self)"""
 
         # XXX since this method is expensive, consider caching the result
@@ -580,12 +580,23 @@
         # dinv is the inverse of self._denominator modulo the prime
         # _PyHASH_MODULUS, or 0 if self._denominator is divisible by
         # _PyHASH_MODULUS.
-        dinv = pow(self._denominator, _PyHASH_MODULUS - 2, _PyHASH_MODULUS)
+
+        # PyPy3: added caching of pow() results for denominators up to 1000
+        denom = self._denominator
+        assert denom >= 0
+        if denom < len(CACHE):
+            dinv = CACHE[denom]
+            if dinv == -1:
+                dinv = pow(denom, _PyHASH_MODULUS - 2, _PyHASH_MODULUS)
+                CACHE[denom] = dinv
+        else:
+            dinv = pow(denom, _PyHASH_MODULUS - 2, _PyHASH_MODULUS)
+
         if not dinv:
             hash_ = _PyHASH_INF
         else:
             hash_ = abs(self._numerator) * dinv % _PyHASH_MODULUS
-        result = hash_ if self >= 0 else -hash_
+        result = hash_ if self._numerator >= 0 else -hash_
         return -2 if result == -1 else result
 
     def __eq__(a, b):

From pypy.commits at gmail.com  Fri Jul  6 14:15:00 2018
From: pypy.commits at gmail.com (arigo)
Date: Fri, 06 Jul 2018 11:15:00 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Fix py3.5 translation of the _cppyy module
Message-ID: <5b3fb1a4.1c69fb81.d65d0.a688@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r94820:bc77e35508e9
Date: 2018-07-06 20:12 +0200
http://bitbucket.org/pypy/pypy/changeset/bc77e35508e9/

Log:	Fix py3.5 translation of the _cppyy module

diff --git a/pypy/module/_cppyy/interp_cppyy.py b/pypy/module/_cppyy/interp_cppyy.py
--- a/pypy/module/_cppyy/interp_cppyy.py
+++ b/pypy/module/_cppyy/interp_cppyy.py
@@ -471,11 +471,8 @@
     __new__ = interp2app(MethodWithProps.descr_method__new__.im_func),
     __call__ = interp2app(MethodWithProps.descr_method_call),
     __get__ = interp2app(MethodWithProps.descr_method_get),
-    im_func = interp_attrproperty_w('w_function', cls=MethodWithProps),
     __func__ = interp_attrproperty_w('w_function', cls=MethodWithProps),
-    im_self = interp_attrproperty_w('w_instance', cls=MethodWithProps),
     __self__ = interp_attrproperty_w('w_instance', cls=MethodWithProps),
-    im_class = interp_attrproperty_w('w_class', cls=MethodWithProps),
     __getattribute__ = interp2app(MethodWithProps.descr_method_getattribute),
     __eq__ = interp2app(MethodWithProps.descr_method_eq),
     __ne__ = descr_generic_ne,
@@ -510,9 +507,9 @@
                             not space.is_w(w_obj, space.w_None) or
                             space.is_w(w_cls, space.type(space.w_None)))
         if asking_for_bound:
-            return MethodWithProps(space, self, w_obj, w_cls)
+            return MethodWithProps(space, self, w_obj)
         else:
-            return MethodWithProps(space, self, None, w_cls)
+            return self   # unbound methods don't exist in Python 3
 
     @unwrap_spec(args_w='args_w')
     def call_args(self, args_w):
@@ -600,7 +597,7 @@
 
     def getname(self, space): 
         # for the benefit of Method/instancemethod
-        return capi.c_method_name(space, self.functions[0].cppmethod)
+        return capi.c_method_name(space, self.functions[0].cppmethod).decode('latin-1')
 
     def __repr__(self):
         return "W_CPPOverload(%s)" % [f.prototype() for f in self.functions]
@@ -626,7 +623,7 @@
             # onto a class and w_this should be set
             cppinstance = self.space.interp_w(W_CPPInstance, w_obj)
             if cppinstance.clsdecl.handle != self.scope.handle:
-	        return MethodWithProps(self.space, self, w_obj, w_cls)    # bound
+	        return MethodWithProps(self.space, self, w_obj)    # bound
         return self      # unbound
 
     @unwrap_spec(args_w='args_w')
@@ -835,7 +832,7 @@
         return self.getitem_impl(self.name, args_w)
 
     def getname(self, space):
-        return self.name
+        return self.name.decode('latin-1')
 
     def __repr__(self):
         return "W_CPPTemplateOverload(%s)" % [f.prototype() for f in self.functions]
@@ -893,7 +890,7 @@
         return self.getitem_impl(self.name, args_w)
 
     def getname(self, space):
-        return self.name
+        return self.name.decode('latin-1')
 
     def __repr__(self):
         return "W_CPPTemplateStaticOverload(%s)" % [f.prototype() for f in self.functions]

From pypy.commits at gmail.com  Sat Jul  7 11:37:51 2018
From: pypy.commits at gmail.com (arigo)
Date: Sat, 07 Jul 2018 08:37:51 -0700 (PDT)
Subject: [pypy-commit] cffi default: Bump version to 1.12.0 (but not
 planning a release soon)
Message-ID: <5b40de4f.1c69fb81.5daa3.5af9@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r3126:7ca82ef0192e
Date: 2018-07-07 17:37 +0200
http://bitbucket.org/cffi/cffi/changeset/7ca82ef0192e/

Log:	Bump version to 1.12.0 (but not planning a release soon)

diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c
--- a/c/_cffi_backend.c
+++ b/c/_cffi_backend.c
@@ -2,7 +2,7 @@
 #include <Python.h>
 #include "structmember.h"
 
-#define CFFI_VERSION  "1.11.5"
+#define CFFI_VERSION  "1.12.0"
 
 #ifdef MS_WIN32
 #include <windows.h>
diff --git a/c/test_c.py b/c/test_c.py
--- a/c/test_c.py
+++ b/c/test_c.py
@@ -12,7 +12,7 @@
 # ____________________________________________________________
 
 import sys
-assert __version__ == "1.11.5", ("This test_c.py file is for testing a version"
+assert __version__ == "1.12.0", ("This test_c.py file is for testing a version"
                                  " of cffi that differs from the one that we"
                                  " get from 'import _cffi_backend'")
 if sys.version_info < (3,):
diff --git a/cffi/__init__.py b/cffi/__init__.py
--- a/cffi/__init__.py
+++ b/cffi/__init__.py
@@ -4,8 +4,8 @@
 from .api import FFI
 from .error import CDefError, FFIError, VerificationError, VerificationMissing
 
-__version__ = "1.11.5"
-__version_info__ = (1, 11, 5)
+__version__ = "1.12.0"
+__version_info__ = (1, 12, 0)
 
 # The verifier module file names are based on the CRC32 of a string that
 # contains the following version number.  It may be older than __version__
diff --git a/cffi/_embedding.h b/cffi/_embedding.h
--- a/cffi/_embedding.h
+++ b/cffi/_embedding.h
@@ -221,7 +221,7 @@
 
         if (f != NULL && f != Py_None) {
             PyFile_WriteString("\nFrom: " _CFFI_MODULE_NAME
-                               "\ncompiled with cffi version: 1.11.5"
+                               "\ncompiled with cffi version: 1.12.0"
                                "\n_cffi_backend module: ", f);
             modules = PyImport_GetModuleDict();
             mod = PyDict_GetItemString(modules, "_cffi_backend");
diff --git a/doc/source/conf.py b/doc/source/conf.py
--- a/doc/source/conf.py
+++ b/doc/source/conf.py
@@ -45,9 +45,9 @@
 # built documents.
 #
 # The short X.Y version.
-version = '1.11'
+version = '1.12'
 # The full version, including alpha/beta/rc tags.
-release = '1.11.5'
+release = '1.12.0'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.
diff --git a/doc/source/installation.rst b/doc/source/installation.rst
--- a/doc/source/installation.rst
+++ b/doc/source/installation.rst
@@ -53,13 +53,13 @@
 
 * https://pypi.python.org/pypi/cffi
 
-* Checksums of the "source" package version 1.11.5:
+* Checksums of the "source" package version 1.12.0:
 
-   - MD5: ac8492f4ad952360737413e82d661908
+   - MD5: ...
 
-   - SHA: 1686e6689a691414d3d22626c837adeee3996dd9
+   - SHA: ...
 
-   - SHA256: e90f17980e6ab0f3c2f3730e56d1fe9bcba1891eeea58966e89d352492cc74f4
+   - SHA256: ...
 
 * Or grab the most current version from the `Bitbucket page`_:
   ``hg clone https://bitbucket.org/cffi/cffi``
diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -190,7 +190,7 @@
 
 `Mailing list <https://groups.google.com/forum/#!forum/python-cffi>`_
 """,
-        version='1.11.5',
+        version='1.12.0',
         packages=['cffi'] if cpython else [],
         package_data={'cffi': ['_cffi_include.h', 'parse_c_type.h', 
                                '_embedding.h', '_cffi_errors.h']}

From pypy.commits at gmail.com  Sat Jul  7 11:39:09 2018
From: pypy.commits at gmail.com (arigo)
Date: Sat, 07 Jul 2018 08:39:09 -0700 (PDT)
Subject: [pypy-commit] cffi default: Last version where we still give this
 warning, thinking about turning it
Message-ID: <5b40de9d.1c69fb81.206d8.daf8@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r3127:30d95e2748ec
Date: 2018-07-07 17:38 +0200
http://bitbucket.org/cffi/cffi/changeset/30d95e2748ec/

Log:	Last version where we still give this warning, thinking about
	turning it into an error now

diff --git a/c/test_c.py b/c/test_c.py
--- a/c/test_c.py
+++ b/c/test_c.py
@@ -3946,7 +3946,7 @@
 
 def test_char_pointer_conversion():
     import warnings
-    assert __version__.startswith(("1.8", "1.9", "1.10", "1.11")), (
+    assert __version__.startswith(("1.8", "1.9", "1.10", "1.11", "1.12")), (
         "consider turning the warning into an error")
     BCharP = new_pointer_type(new_primitive_type("char"))
     BIntP = new_pointer_type(new_primitive_type("int"))

From pypy.commits at gmail.com  Sat Jul  7 11:48:47 2018
From: pypy.commits at gmail.com (arigo)
Date: Sat, 07 Jul 2018 08:48:47 -0700 (PDT)
Subject: [pypy-commit] pypy default: update to cffi/30d95e2748ec
Message-ID: <5b40e0df.1c69fb81.7a562.0dce@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r94822:89ffd6dc4ce1
Date: 2018-07-07 17:47 +0200
http://bitbucket.org/pypy/pypy/changeset/89ffd6dc4ce1/

Log:	update to cffi/30d95e2748ec

diff --git a/lib_pypy/cffi.egg-info/PKG-INFO b/lib_pypy/cffi.egg-info/PKG-INFO
--- a/lib_pypy/cffi.egg-info/PKG-INFO
+++ b/lib_pypy/cffi.egg-info/PKG-INFO
@@ -1,6 +1,6 @@
 Metadata-Version: 1.1
 Name: cffi
-Version: 1.11.5
+Version: 1.12.0
 Summary: Foreign Function Interface for Python calling C code.
 Home-page: http://cffi.readthedocs.org
 Author: Armin Rigo, Maciej Fijalkowski
diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py
--- a/lib_pypy/cffi/__init__.py
+++ b/lib_pypy/cffi/__init__.py
@@ -4,8 +4,8 @@
 from .api import FFI
 from .error import CDefError, FFIError, VerificationError, VerificationMissing
 
-__version__ = "1.11.5"
-__version_info__ = (1, 11, 5)
+__version__ = "1.12.0"
+__version_info__ = (1, 12, 0)
 
 # The verifier module file names are based on the CRC32 of a string that
 # contains the following version number.  It may be older than __version__
diff --git a/lib_pypy/cffi/_embedding.h b/lib_pypy/cffi/_embedding.h
--- a/lib_pypy/cffi/_embedding.h
+++ b/lib_pypy/cffi/_embedding.h
@@ -221,7 +221,7 @@
 
         if (f != NULL && f != Py_None) {
             PyFile_WriteString("\nFrom: " _CFFI_MODULE_NAME
-                               "\ncompiled with cffi version: 1.11.5"
+                               "\ncompiled with cffi version: 1.12.0"
                                "\n_cffi_backend module: ", f);
             modules = PyImport_GetModuleDict();
             mod = PyDict_GetItemString(modules, "_cffi_backend");
diff --git a/lib_pypy/cffi/api.py b/lib_pypy/cffi/api.py
--- a/lib_pypy/cffi/api.py
+++ b/lib_pypy/cffi/api.py
@@ -96,18 +96,21 @@
             self.CData, self.CType = backend._get_types()
         self.buffer = backend.buffer
 
-    def cdef(self, csource, override=False, packed=False):
+    def cdef(self, csource, override=False, packed=False, pack=None):
         """Parse the given C source.  This registers all declared functions,
         types, and global variables.  The functions and global variables can
         then be accessed via either 'ffi.dlopen()' or 'ffi.verify()'.
         The types can be used in 'ffi.new()' and other functions.
         If 'packed' is specified as True, all structs declared inside this
         cdef are packed, i.e. laid out without any field alignment at all.
+        Alternatively, 'pack' can be a small integer, and requests for
+        alignment greater than that are ignored (pack=1 is equivalent to
+        packed=True).
         """
-        self._cdef(csource, override=override, packed=packed)
+        self._cdef(csource, override=override, packed=packed, pack=pack)
 
-    def embedding_api(self, csource, packed=False):
-        self._cdef(csource, packed=packed, dllexport=True)
+    def embedding_api(self, csource, packed=False, pack=None):
+        self._cdef(csource, packed=packed, pack=pack, dllexport=True)
         if self._embedding is None:
             self._embedding = ''
 
diff --git a/lib_pypy/cffi/backend_ctypes.py b/lib_pypy/cffi/backend_ctypes.py
--- a/lib_pypy/cffi/backend_ctypes.py
+++ b/lib_pypy/cffi/backend_ctypes.py
@@ -730,7 +730,8 @@
         return self._new_struct_or_union('union', name, ctypes.Union)
 
     def complete_struct_or_union(self, CTypesStructOrUnion, fields, tp,
-                                 totalsize=-1, totalalignment=-1, sflags=0):
+                                 totalsize=-1, totalalignment=-1, sflags=0,
+                                 pack=0):
         if totalsize >= 0 or totalalignment >= 0:
             raise NotImplementedError("the ctypes backend of CFFI does not support "
                                       "structures completed by verify(); please "
@@ -751,6 +752,8 @@
                 bfield_types[fname] = Ellipsis
         if sflags & 8:
             struct_or_union._pack_ = 1
+        elif pack:
+            struct_or_union._pack_ = pack
         struct_or_union._fields_ = cfields
         CTypesStructOrUnion._bfield_types = bfield_types
         #
diff --git a/lib_pypy/cffi/cparser.py b/lib_pypy/cffi/cparser.py
--- a/lib_pypy/cffi/cparser.py
+++ b/lib_pypy/cffi/cparser.py
@@ -306,11 +306,25 @@
             msg = 'parse error\n%s' % (msg,)
         raise CDefError(msg)
 
-    def parse(self, csource, override=False, packed=False, dllexport=False):
+    def parse(self, csource, override=False, packed=False, pack=None,
+                    dllexport=False):
+        if packed:
+            if packed != True:
+                raise ValueError("'packed' should be False or True; use "
+                                 "'pack' to give another value")
+            if pack:
+                raise ValueError("cannot give both 'pack' and 'packed'")
+            pack = 1
+        elif pack:
+            if pack & (pack - 1):
+                raise ValueError("'pack' must be a power of two, not %r" %
+                    (pack,))
+        else:
+            pack = 0
         prev_options = self._options
         try:
             self._options = {'override': override,
-                             'packed': packed,
+                             'packed': pack,
                              'dllexport': dllexport}
             self._internal_parse(csource)
         finally:
diff --git a/lib_pypy/cffi/model.py b/lib_pypy/cffi/model.py
--- a/lib_pypy/cffi/model.py
+++ b/lib_pypy/cffi/model.py
@@ -342,7 +342,7 @@
     fixedlayout = None
     completed = 0
     partial = False
-    packed = False
+    packed = 0
 
     def __init__(self, name, fldnames, fldtypes, fldbitsize, fldquals=None):
         self.name = name
@@ -414,11 +414,14 @@
             fldtypes = [tp.get_cached_btype(ffi, finishlist)
                         for tp in self.fldtypes]
             lst = list(zip(self.fldnames, fldtypes, self.fldbitsize))
-            sflags = 0
+            extra_flags = ()
             if self.packed:
-                sflags = 8    # SF_PACKED
+                if self.packed == 1:
+                    extra_flags = (8,)    # SF_PACKED
+                else:
+                    extra_flags = (0, self.packed)
             ffi._backend.complete_struct_or_union(BType, lst, self,
-                                                  -1, -1, sflags)
+                                                  -1, -1, *extra_flags)
             #
         else:
             fldtypes = []
diff --git a/lib_pypy/cffi/recompiler.py b/lib_pypy/cffi/recompiler.py
--- a/lib_pypy/cffi/recompiler.py
+++ b/lib_pypy/cffi/recompiler.py
@@ -893,6 +893,12 @@
             else:
                 flags.append("_CFFI_F_CHECK_FIELDS")
             if tp.packed:
+                if tp.packed > 1:
+                    raise NotImplementedError(
+                        "%r is declared with 'pack=%r'; only 0 or 1 are "
+                        "supported in API mode (try to use \"...;\", which "
+                        "does not require a 'pack' declaration)" %
+                        (tp, tp.packed))
                 flags.append("_CFFI_F_PACKED")
         else:
             flags.append("_CFFI_F_EXTERNAL")
diff --git a/pypy/module/_cffi_backend/__init__.py b/pypy/module/_cffi_backend/__init__.py
--- a/pypy/module/_cffi_backend/__init__.py
+++ b/pypy/module/_cffi_backend/__init__.py
@@ -3,7 +3,7 @@
 from rpython.rlib import rdynload, clibffi
 from rpython.rtyper.lltypesystem import rffi
 
-VERSION = "1.11.5"
+VERSION = "1.12.0"
 
 FFI_DEFAULT_ABI = clibffi.FFI_DEFAULT_ABI
 try:
diff --git a/pypy/module/_cffi_backend/newtype.py b/pypy/module/_cffi_backend/newtype.py
--- a/pypy/module/_cffi_backend/newtype.py
+++ b/pypy/module/_cffi_backend/newtype.py
@@ -258,6 +258,11 @@
 SF_PACKED             = 0x08
 SF_STD_FIELD_POS      = 0x80
 
+if sys.platform == 'win32':
+    SF_DEFAULT_PACKING = 8
+else:
+    SF_DEFAULT_PACKING = 0x40000000    # a huge power of two
+
 
 if sys.platform == 'win32':
     DEFAULT_SFLAGS_PLATFORM = SF_MSVC_BITFIELDS
@@ -309,10 +314,18 @@
         w_ctype._custom_field_pos = True
 
 @unwrap_spec(w_ctype=ctypeobj.W_CType, totalsize=int, totalalignment=int,
-             sflags=int)
+             sflags=int, pack=int)
 def complete_struct_or_union(space, w_ctype, w_fields, w_ignored=None,
-                             totalsize=-1, totalalignment=-1, sflags=0):
+                             totalsize=-1, totalalignment=-1, sflags=0,
+                             pack=0):
     sflags = complete_sflags(sflags)
+    if sflags & SF_PACKED:
+        pack = 1
+    elif pack <= 0:
+        pack = SF_DEFAULT_PACKING
+    else:
+        sflags |= SF_PACKED
+
     if (not isinstance(w_ctype, ctypestruct.W_CTypeStructOrUnion)
             or w_ctype.size >= 0):
         raise oefmt(space.w_TypeError,
@@ -362,7 +375,7 @@
         # update the total alignment requirement, but skip it if the
         # field is an anonymous bitfield or if SF_PACKED
         falignorg = ftype.alignof()
-        falign = 1 if sflags & SF_PACKED else falignorg
+        falign = min(pack, falignorg)
         do_align = True
         if (sflags & SF_GCC_ARM_BITFIELDS) == 0 and fbitsize >= 0:
             if (sflags & SF_MSVC_BITFIELDS) == 0:
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -1,7 +1,7 @@
 # ____________________________________________________________
 
 import sys
-assert __version__ == "1.11.5", ("This test_c.py file is for testing a version"
+assert __version__ == "1.12.0", ("This test_c.py file is for testing a version"
                                  " of cffi that differs from the one that we"
                                  " get from 'import _cffi_backend'")
 if sys.version_info < (3,):
@@ -3541,30 +3541,50 @@
     BLong = new_primitive_type("long")
     BChar = new_primitive_type("char")
     BShort = new_primitive_type("short")
-    BStruct = new_struct_type("struct foo")
-    complete_struct_or_union(BStruct, [('a1', BLong, -1),
-                                       ('a2', BChar, -1),
-                                       ('a3', BShort, -1)],
-                             None, -1, -1, SF_PACKED)
-    d = BStruct.fields
-    assert len(d) == 3
-    assert d[0][0] == 'a1'
-    assert d[0][1].type is BLong
+    for extra_args in [(SF_PACKED,), (0, 1)]:
+        BStruct = new_struct_type("struct foo")
+        complete_struct_or_union(BStruct, [('a1', BLong, -1),
+                                           ('a2', BChar, -1),
+                                           ('a3', BShort, -1)],
+                                 None, -1, -1, *extra_args)
+        d = BStruct.fields
+        assert len(d) == 3
+        assert d[0][0] == 'a1'
+        assert d[0][1].type is BLong
+        assert d[0][1].offset == 0
+        assert d[0][1].bitshift == -1
+        assert d[0][1].bitsize == -1
+        assert d[1][0] == 'a2'
+        assert d[1][1].type is BChar
+        assert d[1][1].offset == sizeof(BLong)
+        assert d[1][1].bitshift == -1
+        assert d[1][1].bitsize == -1
+        assert d[2][0] == 'a3'
+        assert d[2][1].type is BShort
+        assert d[2][1].offset == sizeof(BLong) + sizeof(BChar)
+        assert d[2][1].bitshift == -1
+        assert d[2][1].bitsize == -1
+        assert sizeof(BStruct) == sizeof(BLong) + sizeof(BChar) + sizeof(BShort)
+        assert alignof(BStruct) == 1
+    #
+    BStruct2 = new_struct_type("struct foo")
+    complete_struct_or_union(BStruct2, [('b1', BChar, -1),
+                                        ('b2', BLong, -1)],
+                             None, -1, -1, 0, 2)
+    d = BStruct2.fields
+    assert len(d) == 2
+    assert d[0][0] == 'b1'
+    assert d[0][1].type is BChar
     assert d[0][1].offset == 0
     assert d[0][1].bitshift == -1
     assert d[0][1].bitsize == -1
-    assert d[1][0] == 'a2'
-    assert d[1][1].type is BChar
-    assert d[1][1].offset == sizeof(BLong)
+    assert d[1][0] == 'b2'
+    assert d[1][1].type is BLong
+    assert d[1][1].offset == 2
     assert d[1][1].bitshift == -1
     assert d[1][1].bitsize == -1
-    assert d[2][0] == 'a3'
-    assert d[2][1].type is BShort
-    assert d[2][1].offset == sizeof(BLong) + sizeof(BChar)
-    assert d[2][1].bitshift == -1
-    assert d[2][1].bitsize == -1
-    assert sizeof(BStruct) == sizeof(BLong) + sizeof(BChar) + sizeof(BShort)
-    assert alignof(BStruct) == 1
+    assert sizeof(BStruct2) == 2 + sizeof(BLong)
+    assert alignof(BStruct2) == 2
 
 def test_packed_with_bitfields():
     if sys.platform == "win32":
@@ -3915,7 +3935,7 @@
 
 def test_char_pointer_conversion():
     import warnings
-    assert __version__.startswith(("1.8", "1.9", "1.10", "1.11")), (
+    assert __version__.startswith(("1.8", "1.9", "1.10", "1.11", "1.12")), (
         "consider turning the warning into an error")
     BCharP = new_pointer_type(new_primitive_type("char"))
     BIntP = new_pointer_type(new_primitive_type("int"))
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py b/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py
@@ -1824,19 +1824,39 @@
         ffi = FFI(backend=self.Backend())
         ffi.cdef("struct nonpacked { char a; int b; };")
         ffi.cdef("struct is_packed { char a; int b; };", packed=True)
+        ffi.cdef("struct is_packed1 { char a; int b; };", pack=1)
+        ffi.cdef("struct is_packed2 { char a; int b; };", pack=2)
+        ffi.cdef("struct is_packed4 { char a; int b; };", pack=4)
+        ffi.cdef("struct is_packed8 { char a; int b; };", pack=8)
         assert ffi.sizeof("struct nonpacked") == 8
         assert ffi.sizeof("struct is_packed") == 5
+        assert ffi.sizeof("struct is_packed1") == 5
+        assert ffi.sizeof("struct is_packed2") == 6
+        assert ffi.sizeof("struct is_packed4") == 8
+        assert ffi.sizeof("struct is_packed8") == 8
         assert ffi.alignof("struct nonpacked") == 4
         assert ffi.alignof("struct is_packed") == 1
-        s = ffi.new("struct is_packed[2]")
-        s[0].b = 42623381
-        s[0].a = b'X'
-        s[1].b = -4892220
-        s[1].a = b'Y'
-        assert s[0].b == 42623381
-        assert s[0].a == b'X'
-        assert s[1].b == -4892220
-        assert s[1].a == b'Y'
+        assert ffi.alignof("struct is_packed1") == 1
+        assert ffi.alignof("struct is_packed2") == 2
+        assert ffi.alignof("struct is_packed4") == 4
+        assert ffi.alignof("struct is_packed8") == 4
+        for name in ['is_packed', 'is_packed1', 'is_packed2',
+                     'is_packed4', 'is_packed8']:
+            s = ffi.new("struct %s[2]" % name)
+            s[0].b = 42623381
+            s[0].a = b'X'
+            s[1].b = -4892220
+            s[1].a = b'Y'
+            assert s[0].b == 42623381
+            assert s[0].a == b'X'
+            assert s[1].b == -4892220
+            assert s[1].a == b'Y'
+
+    def test_pack_valueerror(self):
+        ffi = FFI(backend=self.Backend())
+        py.test.raises(ValueError, ffi.cdef, "", pack=3)
+        py.test.raises(ValueError, ffi.cdef, "", packed=2)
+        py.test.raises(ValueError, ffi.cdef, "", packed=True, pack=1)
 
     def test_define_integer_constant(self):
         ffi = FFI(backend=self.Backend())
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
@@ -2243,6 +2243,12 @@
         "'API mode' and non-variadic (i.e. declared inside ffibuilder.cdef()"
         "+ffibuilder.set_source() and not taking a final '...' argument)")
 
+def test_pack_not_supported():
+    ffi = FFI()
+    ffi.cdef("""struct foo { char y; int x; };""", pack=2)
+    py.test.raises(NotImplementedError, verify,
+                   ffi, "test_pack_not_supported", "")
+
 def test_gcc_visibility_hidden():
     if sys.platform == 'win32':
         py.test.skip("test for gcc/clang")

From pypy.commits at gmail.com  Sun Jul  8 00:17:42 2018
From: pypy.commits at gmail.com (mattip)
Date: Sat, 07 Jul 2018 21:17:42 -0700 (PDT)
Subject: [pypy-commit] pypy unicode-utf8-py3: repurpose realunicode_w to
 differentiate between bytes and str
Message-ID: <5b419066.1c69fb81.69462.b179@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: unicode-utf8-py3
Changeset: r94823:a50ac22defed
Date: 2018-07-02 14:40 -0500
http://bitbucket.org/pypy/pypy/changeset/a50ac22defed/

Log:	repurpose realunicode_w to differentiate between bytes and str

diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -1716,7 +1716,7 @@
         return w_obj.convert_to_w_unicode(self)
 
     def realunicode_w(self, w_obj):
-        return w_obj.utf8_w(self).decode('utf8')
+        return w_obj.realunicode_w(self)
 
     def utf8_0_w(self, w_obj):
         "Like utf8_w, but rejects strings with NUL bytes."
diff --git a/pypy/interpreter/unicodehelper.py b/pypy/interpreter/unicodehelper.py
--- a/pypy/interpreter/unicodehelper.py
+++ b/pypy/interpreter/unicodehelper.py
@@ -110,8 +110,8 @@
         # instead
         from pypy.module._codecs.locale import (
             unicode_encode_locale_surrogateescape)
-        uni = space.utf8_w(w_uni)
-        if b'\x00' in uni:
+        uni = space.realunicode_w(w_uni)
+        if u'\x00' in uni:
             raise oefmt(space.w_ValueError, "embedded null character")
         bytes = unicode_encode_locale_surrogateescape(
             uni, errorhandler=encode_error_handler(space))
diff --git a/pypy/module/_csv/interp_reader.py b/pypy/module/_csv/interp_reader.py
--- a/pypy/module/_csv/interp_reader.py
+++ b/pypy/module/_csv/interp_reader.py
@@ -73,7 +73,7 @@
                             break
                 raise
             self.line_num += 1
-            line = space.utf8_w(w_line)
+            line = space.realunicode_w(w_line)
             for c in line:
                 if c == b'\0':
                     raise self.error(u"line contains NULL byte")
diff --git a/pypy/objspace/fake/objspace.py b/pypy/objspace/fake/objspace.py
--- a/pypy/objspace/fake/objspace.py
+++ b/pypy/objspace/fake/objspace.py
@@ -218,6 +218,7 @@
     def newutf8(self, x, l):
         return w_some_obj()
 
+    @specialize.argtype(1)
     def newtext(self, x):
         return w_some_obj()
     newtext_or_none = newtext
diff --git a/pypy/objspace/std/bytesobject.py b/pypy/objspace/std/bytesobject.py
--- a/pypy/objspace/std/bytesobject.py
+++ b/pypy/objspace/std/bytesobject.py
@@ -418,6 +418,10 @@
     def bytes_w(self, space):
         return self._value
 
+    def realunicode_w(self, space):
+        raise oefmt(space.w_TypeError,
+                    "unicode object expected, received bytes instead")
+
     def utf8_w(self, space):
         # Use the default encoding.                                             
         encoding = getdefaultencoding(space)
diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -54,6 +54,10 @@
         """representation for debugging purposes"""
         return "%s(%r)" % (self.__class__.__name__, self._utf8)
 
+    def unwrap(self, space):
+        # for testing
+        return self.realunicode_w(space)
+
     def is_w(self, space, w_other):
         if not isinstance(w_other, W_UnicodeObject):
             return False
@@ -87,20 +91,8 @@
     def utf8_w(self, space):
         return self._utf8
 
-    def text_w(self, space):
-        try:
-            identifier = jit.conditional_call_elidable(
-                                self._utf8, g_encode_utf8, self._length)
-        except SurrogateError as e:
-            raise OperationError(space.w_UnicodeEncodeError,
-                    space.newtuple([space.newtext('utf-8'),
-                                    self,
-                                    space.newint(e.index-1),
-                                    space.newint(e.index),
-                                    space.newtext("surrogates not allowed")]))
-        if not jit.isconstant(self):
-            self._utf8 = identifier
-        return identifier
+    def realunicode_w(self, space):
+        return self._utf8.decode('utf8')
 
     def listview_utf8(self):
         assert self.is_ascii()

From pypy.commits at gmail.com  Sun Jul  8 00:17:45 2018
From: pypy.commits at gmail.com (mattip)
Date: Sat, 07 Jul 2018 21:17:45 -0700 (PDT)
Subject: [pypy-commit] pypy unicode-utf8-py3: more places where the rpython
 type needs to be true unicode
Message-ID: <5b419069.1c69fb81.a2cd.5c94@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: unicode-utf8-py3
Changeset: r94824:00be052b857f
Date: 2018-07-02 15:38 -0500
http://bitbucket.org/pypy/pypy/changeset/00be052b857f/

Log:	more places where the rpython type needs to be true unicode

diff --git a/pypy/interpreter/error.py b/pypy/interpreter/error.py
--- a/pypy/interpreter/error.py
+++ b/pypy/interpreter/error.py
@@ -507,9 +507,9 @@
                     if fmt == 'd':
                         result = str(value).decode('ascii')
                     elif fmt == 'R':
-                        result = space.utf8_w(space.repr(value))
+                        result = space.realunicode_w(space.repr(value))
                     elif fmt == 'S':
-                        result = space.utf8_w(space.str(value))
+                        result = space.realunicode_w(space.str(value))
                     elif fmt == 'T':
                         result = _decode_utf8(space.type(value).name)
                     elif fmt == 'N':
diff --git a/pypy/interpreter/gateway.py b/pypy/interpreter/gateway.py
--- a/pypy/interpreter/gateway.py
+++ b/pypy/interpreter/gateway.py
@@ -327,7 +327,7 @@
         self.run_args.append("space.text0_w(%s)" % (self.scopenext(),))
 
     def visit_unicode(self, typ):
-        self.run_args.append("space.utf_8(%s)" % (self.scopenext(),))
+        self.run_args.append("space.realunicode_w(%s)" % (self.scopenext(),))
 
     def visit_utf8(self, typ):
         self.run_args.append("space.utf8_w(%s)" % (self.scopenext(),))
@@ -498,7 +498,7 @@
         self.unwrap.append("space.text_w(%s)" % (self.nextarg(),))
 
     def visit_unicode(self, typ):
-        self.unwrap.append("space.utf_8(%s)" % (self.nextarg(),))
+        self.unwrap.append("space.realunicode_w(%s)" % (self.nextarg(),))
 
     def visit_text0(self, typ):
         self.unwrap.append("space.text0_w(%s)" % (self.nextarg(),))
diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py
--- a/pypy/interpreter/pyopcode.py
+++ b/pypy/interpreter/pyopcode.py
@@ -1649,7 +1649,7 @@
         lst = []
         for i in range(itemcount-1, -1, -1):
             w_item = self.peekvalue(i)
-            lst.append(space.utf8_w(w_item))
+            lst.append(space.realunicode_w(w_item))
         self.dropvalues(itemcount)
         w_res = space.newtext(u''.join(lst))
         self.pushvalue(w_res)
diff --git a/pypy/module/_csv/interp_csv.py b/pypy/module/_csv/interp_csv.py
--- a/pypy/module/_csv/interp_csv.py
+++ b/pypy/module/_csv/interp_csv.py
@@ -43,7 +43,7 @@
     if w_src is None:
         return default
     try:
-        return space.utf8_w(w_src)
+        return space.realunicode_w(w_src)
     except OperationError as e:
         if e.match(space, space.w_TypeError):
             raise oefmt(space.w_TypeError, '"%s" must be a string', attrname)
@@ -56,7 +56,7 @@
         return u'\0'
     if not space.isinstance_w(w_src, space.w_unicode):
         raise oefmt(space.w_TypeError, '"%s" must be string, not %T', name, w_src)
-    src = space.utf8_w(w_src)
+    src = space.realunicode_w(w_src)
     if len(src) == 1:
         return src[0]
     if len(src) == 0:
diff --git a/pypy/module/_csv/interp_writer.py b/pypy/module/_csv/interp_writer.py
--- a/pypy/module/_csv/interp_writer.py
+++ b/pypy/module/_csv/interp_writer.py
@@ -42,9 +42,9 @@
             if space.is_w(w_field, space.w_None):
                 field = u""
             elif space.isinstance_w(w_field, space.w_float):
-                field = space.utf8_w(space.repr(w_field))
+                field = space.realunicode_w(space.repr(w_field))
             else:
-                field = space.utf8_w(space.str(w_field))
+                field = space.realunicode_w(space.str(w_field))
             #
             if dialect.quoting == QUOTE_NONNUMERIC:
                 try:

From pypy.commits at gmail.com  Sun Jul  8 00:17:47 2018
From: pypy.commits at gmail.com (mattip)
Date: Sat, 07 Jul 2018 21:17:47 -0700 (PDT)
Subject: [pypy-commit] pypy unicode-utf8-py3: re-add text_w to differentiate
 W_Unicode (succeeds) from W_Bytes (fails)
Message-ID: <5b41906b.1c69fb81.5b4b2.f4dc@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: unicode-utf8-py3
Changeset: r94825:0ff9a3ee6b88
Date: 2018-07-07 21:15 -0700
http://bitbucket.org/pypy/pypy/changeset/0ff9a3ee6b88/

Log:	re-add text_w to differentiate W_Unicode (succeeds) from W_Bytes
	(fails)

diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -245,6 +245,9 @@
     def bytes_w(self, space):
         self._typed_unwrap_error(space, "bytes")
 
+    def text_w(self, space):
+        self._typed_unwrap_error(space, "unicode")
+
     def utf8_w(self, space):
         self._typed_unwrap_error(space, "unicode")
 
@@ -1615,18 +1618,20 @@
             an utf-8 encoded rpython string.
         """
         assert w_obj is not None
-        return w_obj.utf8_w(self)
+        return w_obj.text_w(self)
 
     @not_rpython    # tests only; should be replaced with bytes_w or text_w
     def str_w(self, w_obj):
         """
-        if w_obj is unicode, call text_w() (i.e., return the UTF-8-nosg
+        if w_obj is unicode, call utf8_w() (i.e., return the UTF-8-nosg
         encoded string). Else, call bytes_w().
 
         We should kill str_w completely and manually substitute it with
         text_w/bytes_w at all call sites.  It remains for now for tests only.
         """
+        XXX # deprecated, leaving in place for clear errors
         if self.isinstance_w(w_obj, self.w_unicode):
+            # XXX lo text_w, but better to deprecate str_w than to fix this
             return w_obj.text_w(self)
         else:
             return w_obj.bytes_w(self)
diff --git a/pypy/objspace/std/typeobject.py b/pypy/objspace/std/typeobject.py
--- a/pypy/objspace/std/typeobject.py
+++ b/pypy/objspace/std/typeobject.py
@@ -1158,9 +1158,10 @@
     if not space.isinstance_w(w_name, space.w_text):
         raise oefmt(space.w_TypeError,
             "__slots__ items must be strings, not '%T'", w_name)
-    if not _isidentifier(space.utf8_w(w_name)):
+    s = space.utf8_w(w_name)
+    if not _isidentifier(s):
         raise oefmt(space.w_TypeError, "__slots__ must be identifiers")
-    return w_name.text_w(space)
+    return s
 
 def create_all_slots(w_self, hasoldstylebase, w_bestbase, force_new_layout):
     from pypy.interpreter.miscutils import string_sort
diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -88,6 +88,9 @@
             uid = (base << IDTAG_SHIFT) | IDTAG_SPECIAL
         return space.newint(uid)
 
+    def text_w(self, space):
+        return self._utf8
+
     def utf8_w(self, space):
         return self._utf8
 
diff --git a/pypy/tool/pytest/appsupport.py b/pypy/tool/pytest/appsupport.py
--- a/pypy/tool/pytest/appsupport.py
+++ b/pypy/tool/pytest/appsupport.py
@@ -23,11 +23,11 @@
         #    self.path = space.unwrap(space.getattr(self.w_file, space.wrap('__path__')))
         #except OperationError:
         #    self.path = space.unwrap(space.getattr(
-        self.path = py.path.local(space.str_w(self.w_file))
+        self.path = py.path.local(space.utf8_w(self.w_file))
         self.space = space
 
     def fullsource(self):
-        filename = self.space.str_w(self.w_file)
+        filename = self.space.utf8_w(self.w_file)
         source = py.code.Source(py.std.linecache.getlines(filename))
         if source.lines:
             return source

From pypy.commits at gmail.com  Sun Jul  8 00:17:49 2018
From: pypy.commits at gmail.com (mattip)
Date: Sat, 07 Jul 2018 21:17:49 -0700 (PDT)
Subject: [pypy-commit] pypy unicode-utf8-py3: provide a nicer error message
 if tests are run with python3
Message-ID: <5b41906d.1c69fb81.29589.c3a6@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: unicode-utf8-py3
Changeset: r94826:0727cbb8aadd
Date: 2018-07-07 21:16 -0700
http://bitbucket.org/pypy/pypy/changeset/0727cbb8aadd/

Log:	provide a nicer error message if tests are run with python3

diff --git a/rpython/tool/leakfinder.py b/rpython/tool/leakfinder.py
--- a/rpython/tool/leakfinder.py
+++ b/rpython/tool/leakfinder.py
@@ -1,5 +1,10 @@
 import sys, gc
-import cStringIO
+try:
+    import cStringIO
+except ImportError as e:
+    if sys.version_info.major > 2:
+        raise RuntimeError('use python 2 to run tests')
+    raise
 import traceback
 
 # Track allocations to detect memory leaks.

From pypy.commits at gmail.com  Sun Jul  8 00:17:52 2018
From: pypy.commits at gmail.com (mattip)
Date: Sat, 07 Jul 2018 21:17:52 -0700 (PDT)
Subject: [pypy-commit] pypy unicode-utf8-py3: merge py3.5 into branch
Message-ID: <5b419070.1c69fb81.bbee6.ce95@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: unicode-utf8-py3
Changeset: r94827:7cba2470321a
Date: 2018-07-07 21:16 -0700
http://bitbucket.org/pypy/pypy/changeset/7cba2470321a/

Log:	merge py3.5 into branch

diff --git a/lib-python/3/fractions.py b/lib-python/3/fractions.py
--- a/lib-python/3/fractions.py
+++ b/lib-python/3/fractions.py
@@ -566,7 +566,7 @@
         else:
             return Fraction(round(self / shift) * shift)
 
-    def __hash__(self):
+    def __hash__(self, CACHE=[-1] * 1001):
         """hash(self)"""
 
         # XXX since this method is expensive, consider caching the result
@@ -580,12 +580,23 @@
         # dinv is the inverse of self._denominator modulo the prime
         # _PyHASH_MODULUS, or 0 if self._denominator is divisible by
         # _PyHASH_MODULUS.
-        dinv = pow(self._denominator, _PyHASH_MODULUS - 2, _PyHASH_MODULUS)
+
+        # PyPy3: added caching of pow() results for denominators up to 1000
+        denom = self._denominator
+        assert denom >= 0
+        if denom < len(CACHE):
+            dinv = CACHE[denom]
+            if dinv == -1:
+                dinv = pow(denom, _PyHASH_MODULUS - 2, _PyHASH_MODULUS)
+                CACHE[denom] = dinv
+        else:
+            dinv = pow(denom, _PyHASH_MODULUS - 2, _PyHASH_MODULUS)
+
         if not dinv:
             hash_ = _PyHASH_INF
         else:
             hash_ = abs(self._numerator) * dinv % _PyHASH_MODULUS
-        result = hash_ if self >= 0 else -hash_
+        result = hash_ if self._numerator >= 0 else -hash_
         return -2 if result == -1 else result
 
     def __eq__(a, b):
diff --git a/lib_pypy/pyrepl/simple_interact.py b/lib_pypy/pyrepl/simple_interact.py
--- a/lib_pypy/pyrepl/simple_interact.py
+++ b/lib_pypy/pyrepl/simple_interact.py
@@ -66,6 +66,10 @@
 
     while 1:
         try:
+            try:
+                sys.stdout.flush()
+            except:
+                pass
             ps1 = getattr(sys, 'ps1', '>>> ')
             ps2 = getattr(sys, 'ps2', '... ')
             try:
diff --git a/pypy/module/__pypy__/__init__.py b/pypy/module/__pypy__/__init__.py
--- a/pypy/module/__pypy__/__init__.py
+++ b/pypy/module/__pypy__/__init__.py
@@ -48,6 +48,7 @@
         'int_lshift':      'interp_intop.int_lshift',
         'int_rshift':      'interp_intop.int_rshift',
         'uint_rshift':     'interp_intop.uint_rshift',
+        'int_mulmod':      'interp_intop.int_mulmod',
     }
 
 
diff --git a/pypy/module/__pypy__/interp_intop.py b/pypy/module/__pypy__/interp_intop.py
--- a/pypy/module/__pypy__/interp_intop.py
+++ b/pypy/module/__pypy__/interp_intop.py
@@ -2,7 +2,7 @@
 from rpython.rtyper.lltypesystem import lltype
 from rpython.rtyper.lltypesystem.lloperation import llop
 from rpython.rlib.rarithmetic import r_uint, intmask
-from rpython.rlib.rarithmetic import int_c_div, int_c_mod
+from rpython.rlib.rarithmetic import int_c_div, int_c_mod, mulmod
 from rpython.rlib import jit
 
 
@@ -39,3 +39,7 @@
     n = r_uint(n)
     x = llop.uint_rshift(lltype.Unsigned, n, m)
     return space.newint(intmask(x))
+
+ at unwrap_spec(a=int, b=int, c=int)
+def int_mulmod(space, a, b, c):
+    return space.newint(mulmod(a, b, c))
diff --git a/pypy/module/__pypy__/test/test_intop.py b/pypy/module/__pypy__/test/test_intop.py
--- a/pypy/module/__pypy__/test/test_intop.py
+++ b/pypy/module/__pypy__/test/test_intop.py
@@ -102,3 +102,7 @@
         assert intop.uint_rshift(-1, 1) == sys.maxsize
         assert intop.uint_rshift(-1, bits-2) == 3
         assert intop.uint_rshift(-1, bits-1) == 1
+
+    def test_mulmod(self):
+        from __pypy__ import intop
+        assert intop.int_mulmod(9373891, 9832739, 2**31-1) == 1025488209
diff --git a/pypy/module/_cppyy/converter.py b/pypy/module/_cppyy/converter.py
--- a/pypy/module/_cppyy/converter.py
+++ b/pypy/module/_cppyy/converter.py
@@ -82,10 +82,9 @@
 
 
 class TypeConverter(object):
-    _immutable_fields_ = ['cffi_name', 'uses_local', 'name']
+    _immutable_fields_ = ['cffi_name', 'name']
 
     cffi_name  = None
-    uses_local = False
     name       = ""
 
     def __init__(self, space, extra):
@@ -108,10 +107,10 @@
         from pypy.module._cppyy.interp_cppyy import FastCallNotPossible
         raise FastCallNotPossible
 
-    def convert_argument(self, space, w_obj, address, call_local):
+    def convert_argument(self, space, w_obj, address):
         self._is_abstract(space)
 
-    def convert_argument_libffi(self, space, w_obj, address, call_local):
+    def convert_argument_libffi(self, space, w_obj, address, scratch):
         from pypy.module._cppyy.interp_cppyy import FastCallNotPossible
         raise FastCallNotPossible
 
@@ -125,10 +124,10 @@
     def to_memory(self, space, w_obj, w_value, offset):
         self._is_abstract(space)
 
-    def finalize_call(self, space, w_obj, call_local):
+    def finalize_call(self, space, w_obj):
         pass
 
-    def free_argument(self, space, arg, call_local):
+    def free_argument(self, space, arg):
         pass
 
 
@@ -172,7 +171,7 @@
         state = space.fromcache(ffitypes.State)
         return state.c_voidp
 
-    def convert_argument(self, space, w_obj, address, call_local):
+    def convert_argument(self, space, w_obj, address):
         w_tc = space.findattr(w_obj, space.newtext('typecode'))
         if w_tc is not None and space.text_w(w_tc) != self.typecode:
             raise oefmt(space.w_TypeError,
@@ -208,7 +207,7 @@
 class NumericTypeConverterMixin(object):
     _mixin_ = True
 
-    def convert_argument_libffi(self, space, w_obj, address, call_local):
+    def convert_argument_libffi(self, space, w_obj, address, scratch):
         x = rffi.cast(self.c_ptrtype, address)
         x[0] = self._unwrap_object(space, w_obj)
 
@@ -228,26 +227,23 @@
 
 class ConstRefNumericTypeConverterMixin(NumericTypeConverterMixin):
     _mixin_ = True
-    _immutable_fields_ = ['uses_local']
-
-    uses_local = True
 
     def cffi_type(self, space):
         state = space.fromcache(ffitypes.State)
         return state.c_voidp
 
-    def convert_argument_libffi(self, space, w_obj, address, call_local):
-        assert rffi.sizeof(self.c_type) <= 2*rffi.sizeof(rffi.VOIDP)  # see interp_cppyy.py
+    def convert_argument_libffi(self, space, w_obj, address, scratch):
         obj = self._unwrap_object(space, w_obj)
-        typed_buf = rffi.cast(self.c_ptrtype, call_local)
+        typed_buf = rffi.cast(self.c_ptrtype, scratch)
         typed_buf[0] = obj
         x = rffi.cast(rffi.VOIDPP, address)
-        x[0] = call_local
+        x[0] = scratch
+
 
 class IntTypeConverterMixin(NumericTypeConverterMixin):
     _mixin_ = True
 
-    def convert_argument(self, space, w_obj, address, call_local):
+    def convert_argument(self, space, w_obj, address):
         x = rffi.cast(self.c_ptrtype, address)
         x[0] = self._unwrap_object(space, w_obj)
         ba = rffi.cast(rffi.CCHARP, address)
@@ -256,7 +252,7 @@
 class FloatTypeConverterMixin(NumericTypeConverterMixin):
     _mixin_ = True
 
-    def convert_argument(self, space, w_obj, address, call_local):
+    def convert_argument(self, space, w_obj, address):
         x = rffi.cast(self.c_ptrtype, address)
         x[0] = self._unwrap_object(space, w_obj)
         ba = rffi.cast(rffi.CCHARP, address)
@@ -273,18 +269,18 @@
         state = space.fromcache(ffitypes.State)
         return state.c_void
 
-    def convert_argument(self, space, w_obj, address, call_local):
+    def convert_argument(self, space, w_obj, address):
         self._is_abstract(space)
 
 
 class BoolConverter(ffitypes.typeid(bool), TypeConverter):
-    def convert_argument(self, space, w_obj, address, call_local):
+    def convert_argument(self, space, w_obj, address):
         x = rffi.cast(rffi.LONGP, address)
         x[0] = self._unwrap_object(space, w_obj)
         ba = rffi.cast(rffi.CCHARP, address)
         ba[capi.c_function_arg_typeoffset(space)] = 'b'
 
-    def convert_argument_libffi(self, space, w_obj, address, call_local):
+    def convert_argument_libffi(self, space, w_obj, address, scratch):
         x = rffi.cast(rffi.LONGP, address)
         x[0] = self._unwrap_object(space, w_obj)
 
@@ -303,13 +299,13 @@
             address[0] = '\x00'
 
 class CharConverter(ffitypes.typeid(rffi.CHAR), TypeConverter):
-    def convert_argument(self, space, w_obj, address, call_local):
+    def convert_argument(self, space, w_obj, address):
         x = rffi.cast(rffi.CCHARP, address)
         x[0] = self._unwrap_object(space, w_obj)
         ba = rffi.cast(rffi.CCHARP, address)
         ba[capi.c_function_arg_typeoffset(space)] = 'b'
 
-    def convert_argument_libffi(self, space, w_obj, address, call_local):
+    def convert_argument_libffi(self, space, w_obj, address, scratch):
         x = rffi.cast(self.c_ptrtype, address)
         x[0] = self._unwrap_object(space, w_obj)
 
@@ -348,7 +344,7 @@
         state = space.fromcache(ffitypes.State)
         return state.c_voidp
 
-    def convert_argument_libffi(self, space, w_obj, address, call_local):
+    def convert_argument_libffi(self, space, w_obj, address, scratch):
         from pypy.module._cppyy.interp_cppyy import FastCallNotPossible
         raise FastCallNotPossible
 
@@ -381,7 +377,7 @@
 
 
 class CStringConverter(TypeConverter):
-    def convert_argument(self, space, w_obj, address, call_local):
+    def convert_argument(self, space, w_obj, address):
         x = rffi.cast(rffi.LONGP, address)
         arg = space.text_w(w_obj)
         x[0] = rffi.cast(rffi.LONG, rffi.str2charp(arg))
@@ -393,7 +389,7 @@
         charpptr = rffi.cast(rffi.CCHARPP, address)
         return space.newtext(rffi.charp2str(charpptr[0]))
 
-    def free_argument(self, space, arg, call_local):
+    def free_argument(self, space, arg):
         lltype.free(rffi.cast(rffi.CCHARPP, arg)[0], flavor='raw')
 
 class CStringConverterWithSize(CStringConverter):
@@ -423,13 +419,13 @@
         state = space.fromcache(ffitypes.State)
         return state.c_voidp
 
-    def convert_argument(self, space, w_obj, address, call_local):
+    def convert_argument(self, space, w_obj, address):
         x = rffi.cast(rffi.VOIDPP, address)
         x[0] = self._unwrap_object(space, w_obj)
         ba = rffi.cast(rffi.CCHARP, address)
         ba[capi.c_function_arg_typeoffset(space)] = 'o'
 
-    def convert_argument_libffi(self, space, w_obj, address, call_local):
+    def convert_argument_libffi(self, space, w_obj, address, scratch):
         x = rffi.cast(rffi.VOIDPP, address)
         x[0] = self._unwrap_object(space, w_obj)
 
@@ -452,37 +448,39 @@
             address[0] = rffi.cast(rffi.VOIDP, self._unwrap_object(space, w_value))
 
 class VoidPtrPtrConverter(TypeConverter):
-    _immutable_fields_ = ['uses_local', 'typecode']
+    typecode = 'p'
 
-    uses_local = True
-    typecode = 'a'
+    def __init__(self, space, extra):
+        self.ref_buffer = lltype.nullptr(rffi.VOIDPP.TO)
 
-    def convert_argument(self, space, w_obj, address, call_local):
+    def convert_argument(self, space, w_obj, address):
         x = rffi.cast(rffi.VOIDPP, address)
-        ba = rffi.cast(rffi.CCHARP, address)
         try:
             x[0] = get_rawbuffer(space, w_obj)
         except TypeError:
-            r = rffi.cast(rffi.VOIDPP, call_local)
-            r[0] = rffi.cast(rffi.VOIDP, get_rawobject(space, w_obj))
-            x[0] = rffi.cast(rffi.VOIDP, call_local)
+            ptr = rffi.cast(rffi.VOIDP, get_rawobject(space, w_obj))
+            self.ref_buffer = lltype.malloc(rffi.VOIDPP.TO, 1, flavor='raw')
+            self.ref_buffer[0] = ptr
+            x[0] = self.ref_buffer
+        ba = rffi.cast(rffi.CCHARP, address)
         ba[capi.c_function_arg_typeoffset(space)] = self.typecode
 
-    def finalize_call(self, space, w_obj, call_local):
-        r = rffi.cast(rffi.VOIDPP, call_local)
-        try:
-            set_rawobject(space, w_obj, r[0])
-        except OperationError:
-            pass             # no set on buffer/array/None
+    def finalize_call(self, space, w_obj):
+        if self.ref_buffer:
+            set_rawobject(space, w_obj, self.ref_buffer[0])
+
+    def free_argument(self, space, arg):
+        if self.ref_buffer:
+            lltype.free(self.ref_buffer, flavor='raw')
+            self.ref_buffer = lltype.nullptr(rffi.VOIDPP.TO)
 
 class VoidPtrRefConverter(VoidPtrPtrConverter):
-    _immutable_fields_ = ['uses_local', 'typecode']
-    uses_local = True
+    _immutable_fields_ = ['typecode']
     typecode   = 'V'
 
 class InstanceRefConverter(TypeConverter):
     _immutable_fields_ = ['typecode', 'clsdecl']
-    typecode    = 'V'
+    typecode = 'V'
 
     def __init__(self, space, clsdecl):
         from pypy.module._cppyy.interp_cppyy import W_CPPClassDecl
@@ -508,14 +506,14 @@
         state = space.fromcache(ffitypes.State)
         return state.c_voidp
 
-    def convert_argument(self, space, w_obj, address, call_local):
+    def convert_argument(self, space, w_obj, address):
         x = rffi.cast(rffi.VOIDPP, address)
         x[0] = rffi.cast(rffi.VOIDP, self._unwrap_object(space, w_obj))
         address = rffi.cast(capi.C_OBJECT, address)
         ba = rffi.cast(rffi.CCHARP, address)
         ba[capi.c_function_arg_typeoffset(space)] = self.typecode
 
-    def convert_argument_libffi(self, space, w_obj, address, call_local):
+    def convert_argument_libffi(self, space, w_obj, address, scratch):
         x = rffi.cast(rffi.VOIDPP, address)
         x[0] = rffi.cast(rffi.VOIDP, self._unwrap_object(space, w_obj))
 
@@ -539,7 +537,7 @@
 
 class InstanceConverter(InstanceRefConverter):
 
-    def convert_argument_libffi(self, space, w_obj, address, call_local):
+    def convert_argument_libffi(self, space, w_obj, address, scratch):
         from pypy.module._cppyy.interp_cppyy import FastCallNotPossible
         raise FastCallNotPossible       # TODO: by-value is a jit_libffi special case
 
@@ -551,9 +549,8 @@
     def to_memory(self, space, w_obj, w_value, offset):
         self._is_abstract(space)
 
-
 class InstancePtrConverter(InstanceRefConverter):
-    typecode    = 'o'
+    typecode = 'o'
 
     def _unwrap_object(self, space, w_obj):
         try:
@@ -574,36 +571,41 @@
         address[0] = rffi.cast(rffi.VOIDP, self._unwrap_object(space, w_value))
 
 class InstancePtrPtrConverter(InstancePtrConverter):
-    _immutable_fields_ = ['uses_local']
+    typecode = 'o'
 
-    uses_local = True
+    def __init__(self, space, extra):
+        InstancePtrConverter.__init__(self, space, extra)
+        self.ref_buffer = lltype.nullptr(rffi.VOIDPP.TO)
 
-    def convert_argument(self, space, w_obj, address, call_local):
-        r = rffi.cast(rffi.VOIDPP, call_local)
-        r[0] = rffi.cast(rffi.VOIDP, self._unwrap_object(space, w_obj))
+    def convert_argument(self, space, w_obj, address):
         x = rffi.cast(rffi.VOIDPP, address)
-        x[0] = rffi.cast(rffi.VOIDP, call_local)
-        address = rffi.cast(capi.C_OBJECT, address)
+        ptr = rffi.cast(rffi.VOIDP, self._unwrap_object(space, w_obj))
+        self.ref_buffer = lltype.malloc(rffi.VOIDPP.TO, 1, flavor='raw')
+        self.ref_buffer[0] = ptr
+        x[0] = self.ref_buffer
         ba = rffi.cast(rffi.CCHARP, address)
-        ba[capi.c_function_arg_typeoffset(space)] = 'o'
+        ba[capi.c_function_arg_typeoffset(space)] = self.typecode
 
-    def convert_argument_libffi(self, space, w_obj, address, call_local):
+    def convert_argument_libffi(self, space, w_obj, address, scratch):
         # TODO: finalize_call not yet called for fast call (see interp_cppyy.py)
         from pypy.module._cppyy.interp_cppyy import FastCallNotPossible
         raise FastCallNotPossible
 
-    def finalize_call(self, space, w_obj, call_local):
-        from pypy.module._cppyy.interp_cppyy import W_CPPInstance
-        assert isinstance(w_obj, W_CPPInstance)
-        r = rffi.cast(rffi.VOIDPP, call_local)
-        w_obj._rawobject = rffi.cast(capi.C_OBJECT, r[0])
-
     def from_memory(self, space, w_obj, w_pycppclass, offset):
         address = rffi.cast(capi.C_OBJECT, self._get_raw_address(space, w_obj, offset))
         from pypy.module._cppyy import interp_cppyy
         return interp_cppyy.wrap_cppinstance(
             space, address, self.clsdecl, do_cast=False, is_ref=True)
 
+    def finalize_call(self, space, w_obj):
+        if self.ref_buffer:
+            set_rawobject(space, w_obj, self.ref_buffer[0])
+
+    def free_argument(self, space, arg):
+        if self.ref_buffer:
+            lltype.free(self.ref_buffer, flavor='raw')
+            self.ref_buffer = lltype.nullptr(rffi.VOIDPP.TO)
+
 class StdStringConverter(InstanceConverter):
 
     def __init__(self, space, extra):
@@ -628,7 +630,7 @@
         except Exception:
             InstanceConverter.to_memory(self, space, w_obj, w_value, offset)
 
-    def free_argument(self, space, arg, call_local):
+    def free_argument(self, space, arg):
         capi.c_destruct(space, self.clsdecl, rffi.cast(capi.C_OBJECT, rffi.cast(rffi.VOIDPP, arg)[0]))
 
 class StdStringRefConverter(InstancePtrConverter):
@@ -646,7 +648,7 @@
         state = space.fromcache(ffitypes.State)
         return state.c_voidp
 
-    def convert_argument(self, space, w_obj, address, call_local):
+    def convert_argument(self, space, w_obj, address):
         if hasattr(space, "fake"):
             raise NotImplementedError
         space.getbuiltinmodule("cpyext")
@@ -657,7 +659,7 @@
         ba = rffi.cast(rffi.CCHARP, address)
         ba[capi.c_function_arg_typeoffset(space)] = 'a'
 
-    def convert_argument_libffi(self, space, w_obj, address, call_local):
+    def convert_argument_libffi(self, space, w_obj, address, scratch):
         # TODO: free_argument not yet called for fast call (see interp_cppyy.py)
         from pypy.module._cppyy.interp_cppyy import FastCallNotPossible
         raise FastCallNotPossible
@@ -671,7 +673,7 @@
         x = rffi.cast(rffi.VOIDPP, address)
         x[0] = rffi.cast(rffi.VOIDP, ref)"""
 
-    def free_argument(self, space, arg, call_local):
+    def free_argument(self, space, arg):
         if hasattr(space, "fake"):
             raise NotImplementedError
         space.getbuiltinmodule("cpyext")
@@ -685,7 +687,7 @@
     def __init__(self, space, signature):
         self.signature = signature
 
-    def convert_argument(self, space, w_obj, address, call_local):
+    def convert_argument(self, space, w_obj, address):
         # TODO: atm, does not actually get an overload, but a staticmethod
         from pypy.module._cppyy.interp_cppyy import W_CPPOverload
         cppol = space.interp_w(W_CPPOverload, w_obj)
@@ -740,7 +742,7 @@
         raise oefmt(space.w_TypeError,
                     "cannot pass %T instance as %s", w_obj, self.rawdecl.name)
 
-    def convert_argument(self, space, w_obj, address, call_local):
+    def convert_argument(self, space, w_obj, address):
         x = rffi.cast(rffi.VOIDPP, address)
         x[0] = rffi.cast(rffi.VOIDP, self._unwrap_object(space, w_obj))
         address = rffi.cast(capi.C_OBJECT, address)
diff --git a/pypy/module/_cppyy/interp_cppyy.py b/pypy/module/_cppyy/interp_cppyy.py
--- a/pypy/module/_cppyy/interp_cppyy.py
+++ b/pypy/module/_cppyy/interp_cppyy.py
@@ -1,6 +1,9 @@
 import pypy.module._cppyy.capi as capi
 
 from pypy.interpreter.error import OperationError, oefmt
+from pypy.interpreter.function import Method
+from pypy.interpreter.argument import Arguments
+from pypy.interpreter.typedef import interp_attrproperty_w, descr_generic_ne, make_weakref_descr
 from pypy.interpreter.gateway import interp2app, unwrap_spec
 from pypy.interpreter.typedef import TypeDef, GetSetProperty, interp_attrproperty
 from pypy.interpreter.baseobjspace import W_Root
@@ -166,11 +169,13 @@
 #  W_CPPConstructorOverload:      constructors
 #  W_CPPStaticOverload:           free and static functions
 #  W_CPPTemplateOverload:         templated methods
-#  W_CPPTemplateStaticOveload:    templated free and static functions
+#  W_CPPTemplateStaticOverload:   templated free and static functions
 #
 #  CPPMethod:         a single function or method (base class)
 #  CPPSetItem:        specialization for Python's __setitem__
 #
+#  MethodWithProps:   python instancemethod that forwards properties
+#
 # All methods/functions derive from CPPMethod and are collected as overload
 # candidates in user-facing overload classes. Templated methods are a two-step
 # process, where first the template is instantiated (or selected if already
@@ -183,13 +188,13 @@
     the memory_regulator."""
 
     _attrs_ = ['space', 'scope', 'cppmethod', 'arg_defs', 'args_required',
-               'converters', 'executor', '_funcaddr', 'cif_descr', 'uses_local']
+               'converters', 'executor', '_funcaddr', 'cif_descr']
     _immutable_fields_ = ['scope', 'cppmethod', 'arg_defs', 'args_required',
-                          'converters', 'executor', 'uses_local']
+               'converters', 'executor', '_funcaddr', 'cif_descr']
 
-    def __init__(self, space, declaring_scope, cppmethod, arg_defs, args_required):
+    def __init__(self, space, decl_scope, cppmethod, arg_defs, args_required):
         self.space = space
-        self.scope = declaring_scope
+        self.scope = decl_scope
         self.cppmethod = cppmethod
         self.arg_defs = arg_defs
         self.args_required = args_required
@@ -200,14 +205,6 @@
         self.executor = None
         self.cif_descr = lltype.nullptr(jit_libffi.CIF_DESCRIPTION)
         self._funcaddr = lltype.nullptr(capi.C_FUNC_PTR.TO)
-        self.uses_local = False
-
-    def _address_from_local_buffer(self, call_local, idx):
-        if not call_local:
-            return call_local
-        stride = 2*rffi.sizeof(rffi.VOIDP)
-        loc_idx = lltype.direct_ptradd(rffi.cast(rffi.CCHARP, call_local), idx*stride)
-        return rffi.cast(rffi.VOIDP, loc_idx)
 
     @jit.unroll_safe
     def call(self, cppthis, args_w, useffi):
@@ -233,61 +230,55 @@
             except Exception:
                 pass
 
-        # some calls, e.g. for ptr-ptr or reference need a local array to store data for
-        # the duration of the call
-        if self.uses_local:
-            call_local = lltype.malloc(rffi.VOIDP.TO, 2*len(args_w), flavor='raw')
-        else:
-            call_local = lltype.nullptr(rffi.VOIDP.TO)
+        # attempt to call directly through ffi chain
+        if useffi and self._funcaddr:
+            try:
+                return self.do_fast_call(cppthis, args_w)
+            except FastCallNotPossible:
+                pass      # can happen if converters or executor does not implement ffi
 
+        # ffi chain must have failed; using stub functions instead
+        args, stat = self.prepare_arguments(args_w)
         try:
-            # attempt to call directly through ffi chain
-            if useffi and self._funcaddr:
-                try:
-                    return self.do_fast_call(cppthis, args_w, call_local)
-                except FastCallNotPossible:
-                    pass      # can happen if converters or executor does not implement ffi
-
-            # ffi chain must have failed; using stub functions instead
-            args, stat = self.prepare_arguments(args_w, call_local)
-            try:
-                result = self.executor.execute(
-                    self.space, self.cppmethod, cppthis, len(args_w), args)
-                if stat[0] != rffi.cast(rffi.ULONG, 0):
-                    what = rffi.cast(rffi.CCHARP, stat[1])
-                    pywhat = rffi.charp2str(what)
-                    capi.c_free(self.space, rffi.cast(rffi.VOIDP, what))
-                    raise OperationError(self.space.w_Exception, self.space.newtext(pywhat))
-                return result
-            finally:
-                self.finalize_call(args, args_w, call_local)
+            result = self.executor.execute(
+                self.space, self.cppmethod, cppthis, len(args_w), args)
+            if stat[0] != rffi.cast(rffi.ULONG, 0):
+                what = rffi.cast(rffi.CCHARP, stat[1])
+                pywhat = rffi.charp2str(what)
+                capi.c_free(self.space, rffi.cast(rffi.VOIDP, what))
+                raise OperationError(self.space.w_Exception, self.space.newtext(pywhat))
+            return result
         finally:
-            if call_local:
-                lltype.free(call_local, flavor='raw')
+            self.finalize_call(args, args_w)
 
     @jit.unroll_safe
-    def do_fast_call(self, cppthis, args_w, call_local):
+    def do_fast_call(self, cppthis, args_w):
         if self.cif_descr == lltype.nullptr(jit_libffi.CIF_DESCRIPTION):
             raise FastCallNotPossible
         jit.promote(self)
         cif_descr = self.cif_descr
-        buffer = lltype.malloc(rffi.CCHARP.TO, cif_descr.exchange_size, flavor='raw')
+        # add extra space for const-ref support (see converter.py)
+        buffer = lltype.malloc(rffi.CCHARP.TO, cif_descr.exchange_size+len(self.arg_defs)*rffi.sizeof(rffi.DOUBLE), flavor='raw')
+        thisoff = 0
         try:
-            # this pointer
-            data = rffi.ptradd(buffer, cif_descr.exchange_args[0])
-            x = rffi.cast(rffi.LONGP, data)       # LONGP needed for test_zjit.py
-            x[0] = rffi.cast(rffi.LONG, cppthis)
+            if cppthis:
+                # this pointer
+                data = rffi.ptradd(buffer, cif_descr.exchange_args[0])
+                x = rffi.cast(rffi.LONGP, data)       # LONGP needed for test_zjit.py
+                x[0] = rffi.cast(rffi.LONG, cppthis)
+                thisoff = 1
 
-            # other arguments and defaults
-            i = len(self.arg_defs) + 1
+            # actual provided arguments
+            i = -1      # needed if all arguments are defaults
             for i in range(len(args_w)):
                 conv = self.converters[i]
-                w_arg = args_w[i]
-                data = rffi.ptradd(buffer, cif_descr.exchange_args[i+1])
-                conv.convert_argument_libffi(self.space, w_arg, data, call_local)
+                data = rffi.ptradd(buffer, cif_descr.exchange_args[i+thisoff])
+                scratch = rffi.ptradd(buffer, cif_descr.exchange_size+i*rffi.sizeof(rffi.DOUBLE))
+                conv.convert_argument_libffi(self.space, args_w[i], data, scratch)
+            # drop in defaults for the rest
             for j in range(i+1, len(self.arg_defs)):
                 conv = self.converters[j]
-                data = rffi.ptradd(buffer, cif_descr.exchange_args[j+1])
+                data = rffi.ptradd(buffer, cif_descr.exchange_args[j+thisoff])
                 conv.default_argument_libffi(self.space, data)
 
             assert self._funcaddr
@@ -346,22 +337,17 @@
         self.executor = executor.get_executor(
             self.space, capi.c_method_result_type(self.space, self.cppmethod))
 
-        for conv in self.converters:
-            if conv.uses_local:
-                self.uses_local = True
-                break
-
         # Each CPPMethod corresponds one-to-one to a C++ equivalent and cppthis
         # has been offset to the matching class. Hence, the libffi pointer is
         # uniquely defined and needs to be setup only once.
         funcaddr = capi.c_function_address(self.space, self.cppmethod)
-        if funcaddr and cppthis:      # TODO: methods only for now
+        if funcaddr:
             state = self.space.fromcache(ffitypes.State)
 
-            # argument type specification (incl. cppthis)
+            # argument type specification (incl. cppthis if applicable)
             fargs = []
             try:
-                fargs.append(state.c_voidp)
+                if cppthis: fargs.append(state.c_voidp)
                 for i, conv in enumerate(self.converters):
                     fargs.append(conv.cffi_type(self.space))
                 fresult = self.executor.cffi_type(self.space)
@@ -386,7 +372,7 @@
             self._funcaddr = funcaddr
 
     @jit.unroll_safe
-    def prepare_arguments(self, args_w, call_local):
+    def prepare_arguments(self, args_w):
         args = capi.c_allocate_function_args(self.space, len(args_w))
         stride = capi.c_function_arg_sizeof(self.space)
         for i in range(len(args_w)):
@@ -394,15 +380,13 @@
             w_arg = args_w[i]
             try:
                 arg_i = lltype.direct_ptradd(rffi.cast(rffi.CCHARP, args), i*stride)
-                loc_i = self._address_from_local_buffer(call_local, i)
-                conv.convert_argument(self.space, w_arg, rffi.cast(capi.C_OBJECT, arg_i), loc_i)
+                conv.convert_argument(self.space, w_arg, rffi.cast(capi.C_OBJECT, arg_i))
             except:
                 # fun :-(
                 for j in range(i):
                     conv = self.converters[j]
                     arg_j = lltype.direct_ptradd(rffi.cast(rffi.CCHARP, args), j*stride)
-                    loc_j = self._address_from_local_buffer(call_local, j)
-                    conv.free_argument(self.space, rffi.cast(capi.C_OBJECT, arg_j), loc_j)
+                    conv.free_argument(self.space, rffi.cast(capi.C_OBJECT, arg_j))
                 capi.c_deallocate_function_args(self.space, args)
                 raise
         stat = rffi.cast(rffi.ULONGP,
@@ -411,14 +395,13 @@
         return args, stat
 
     @jit.unroll_safe
-    def finalize_call(self, args, args_w, call_local):
+    def finalize_call(self, args, args_w):
         stride = capi.c_function_arg_sizeof(self.space)
         for i in range(len(args_w)):
             conv = self.converters[i]
             arg_i = lltype.direct_ptradd(rffi.cast(rffi.CCHARP, args), i*stride)
-            loc_i = self._address_from_local_buffer(call_local, i)
-            conv.finalize_call(self.space, args_w[i], loc_i)
-            conv.free_argument(self.space, rffi.cast(capi.C_OBJECT, arg_i), loc_i)
+            conv.finalize_call(self.space, args_w[i])
+            conv.free_argument(self.space, rffi.cast(capi.C_OBJECT, arg_i))
         capi.c_deallocate_function_args(self.space, args)
 
     def signature(self, show_formalargs=True):
@@ -466,42 +449,78 @@
         CPPMethod.call(self, cppthis, args_w, useffi)
 
 
+# CPPOverloads have settable flags that control memory and ffi behavior. These flags
+# need forwarding, which the normal instancemethod does not provide, hence this
+# derived class.
+class MethodWithProps(Method):
+    # allow user to determine ffi use rules per overload
+    def fget_useffi(self, space):
+	f = space.interp_w(W_CPPOverload, self.w_function)
+        return f.fget_useffi(space)
+
+    @unwrap_spec(value=bool)
+    def fset_useffi(self, space, value):
+        f = space.interp_w(W_CPPOverload, self.w_function)
+        f.fset_useffi(space, value)
+
+MethodWithProps.typedef = TypeDef(
+    "cpp_instancemethod",
+    __doc__ = """cpp_instancemethod(function, instance, class)
+
+Create an instance method object.""",
+    __new__ = interp2app(MethodWithProps.descr_method__new__.im_func),
+    __call__ = interp2app(MethodWithProps.descr_method_call),
+    __get__ = interp2app(MethodWithProps.descr_method_get),
+    __func__ = interp_attrproperty_w('w_function', cls=MethodWithProps),
+    __self__ = interp_attrproperty_w('w_instance', cls=MethodWithProps),
+    __getattribute__ = interp2app(MethodWithProps.descr_method_getattribute),
+    __eq__ = interp2app(MethodWithProps.descr_method_eq),
+    __ne__ = descr_generic_ne,
+    __hash__ = interp2app(MethodWithProps.descr_method_hash),
+    __repr__ = interp2app(MethodWithProps.descr_method_repr),
+    __reduce__ = interp2app(MethodWithProps.descr_method__reduce__),
+    __weakref__ = make_weakref_descr(MethodWithProps),
+    __useffi__ = GetSetProperty(MethodWithProps.fget_useffi, MethodWithProps.fset_useffi),
+    )
+MethodWithProps.typedef.acceptable_as_base_class = False
+
+
 class W_CPPOverload(W_Root):
     """App-level dispatcher: controls a collection of (potentially) overloaded methods
     or functions. Calls these in order and deals with error handling and reporting."""
 
-    _attrs_ = ['space', 'scope', 'functions', 'flags', 'w_this']
+    _attrs_ = ['space', 'scope', 'functions', 'flags']
     _immutable_fields_ = ['scope', 'functions[*]']
 
-    def __init__(self, space, declaring_scope, functions, flags = OVERLOAD_FLAGS_USE_FFI):
+    def __init__(self, space, decl_scope, funcs, flags = OVERLOAD_FLAGS_USE_FFI):
         self.space  = space
-        self.scope  = declaring_scope
+        self.scope  = decl_scope
         from rpython.rlib import debug
-        self.functions = debug.make_sure_not_resized(functions)
+        self.functions = debug.make_sure_not_resized(funcs)
         self.flags  = flags
-        self.w_this = self.space.w_None
+
+    def descr_get(self, w_obj, w_cls=None):
+        """functionobject.__get__(obj[, type]) -> method"""
+        # TODO: check validity of w_cls if given
+        space = self.space
+        asking_for_bound = (space.is_none(w_cls) or
+                            not space.is_w(w_obj, space.w_None) or
+                            space.is_w(w_cls, space.type(space.w_None)))
+        if asking_for_bound:
+            return MethodWithProps(space, self, w_obj)
+        else:
+            return self   # unbound methods don't exist in Python 3
 
     @unwrap_spec(args_w='args_w')
-    def descr_get(self, w_cppinstance, args_w):
-        if self.space.is_w(w_cppinstance, self.space.w_None):
-            return self  # unbound, so no new instance needed
-        cppol = W_CPPOverload(self.space, self.scope, self.functions, self.flags)
-        cppol.w_this = w_cppinstance
-        return cppol     # bound
-
-    @unwrap_spec(args_w='args_w')
-    def call(self, args_w):
-        if self.space.is_w(self.w_this, self.space.w_None) and len(args_w):
-            w_this = args_w[0]
-            args_w = args_w[1:]
-        else:
-            w_this = self.w_this
+    def call_args(self, args_w):
+        jit.promote(self)
+        w_this = args_w[0]
         cppinstance = self.space.interp_w(W_CPPInstance, w_this)
         cppinstance._nullcheck()
         if not capi.c_is_subtype(self.space, cppinstance.clsdecl, self.scope):
             raise oefmt(self.space.w_TypeError,
                 "cannot pass %T instance as %s", w_this, self.scope.name)
-        return self.call_impl(cppinstance.get_cppthis(self.scope), args_w)
+        return self.call_impl(cppinstance.get_cppthis(self.scope), args_w[1:])
 
     @jit.unroll_safe
     def call_impl(self, cppthis, args_w):
@@ -576,13 +595,17 @@
     def fget_doc(self, space):
         return self.prototype()
 
+    def getname(self, space): 
+        # for the benefit of Method/instancemethod
+        return capi.c_method_name(space, self.functions[0].cppmethod).decode('latin-1')
+
     def __repr__(self):
         return "W_CPPOverload(%s)" % [f.prototype() for f in self.functions]
 
 W_CPPOverload.typedef = TypeDef(
     'CPPOverload',
     __get__    = interp2app(W_CPPOverload.descr_get),
-    __call__   = interp2app(W_CPPOverload.call),
+    __call__   = interp2app(W_CPPOverload.call_args),
     __useffi__ = GetSetProperty(W_CPPOverload.fget_useffi, W_CPPOverload.fset_useffi),
     __doc__    = GetSetProperty(W_CPPOverload.fget_doc)
 )
@@ -593,27 +616,24 @@
 class W_CPPStaticOverload(W_CPPOverload):
     _attrs_ = []
 
-    @unwrap_spec(args_w='args_w')
-    def descr_get(self, w_cppinstance, args_w):
-        if isinstance(w_cppinstance, W_CPPInstance):
+    def descr_get(self, w_obj, w_cls=None):
+        if isinstance(w_obj, W_CPPInstance):
             # two possibilities: this is a static function called on an
             # instance and w_this must not be set, or a free function rebound
             # onto a class and w_this should be set
-            cppinstance = self.space.interp_w(W_CPPInstance, w_cppinstance)
+            cppinstance = self.space.interp_w(W_CPPInstance, w_obj)
             if cppinstance.clsdecl.handle != self.scope.handle:
-                cppol = W_CPPStaticOverload(self.space, self.scope, self.functions, self.flags)
-                cppol.w_this = w_cppinstance
-                return cppol       # bound
+	        return MethodWithProps(self.space, self, w_obj)    # bound
         return self      # unbound
 
     @unwrap_spec(args_w='args_w')
-    def call(self, args_w):
-        if not self.space.is_w(self.w_this, self.space.w_None):
-            # free function used as bound method, put self back into args_w
-            cppinstance = self.space.interp_w(W_CPPInstance, self.w_this)
-            cppinstance._nullcheck()
-            args_w = [self.w_this] + args_w
+    def call_args(self, args_w):
+        jit.promote(self)
+        #if isinstance(args_w[0], W_CPPInstance):
+            # free function used as bound method, leave in place
         return self.call_impl(capi.C_NULL_OBJECT, args_w)
+        # free functions are implemented as methods of 'namespace' classes, remove 'instance'
+        #return self.call_impl(capi.C_NULL_OBJECT, args_w[1:])
 
     def __repr__(self):
         return "W_CPPStaticOverload(%s)" % [f.prototype() for f in self.functions]
@@ -621,7 +641,7 @@
 W_CPPStaticOverload.typedef = TypeDef(
     'CPPStaticOverload',
     __get__    = interp2app(W_CPPStaticOverload.descr_get),
-    __call__   = interp2app(W_CPPStaticOverload.call),
+    __call__   = interp2app(W_CPPStaticOverload.call_args),
     __useffi__ = GetSetProperty(W_CPPStaticOverload.fget_useffi, W_CPPStaticOverload.fset_useffi),
     __doc__    = GetSetProperty(W_CPPStaticOverload.fget_doc)
 )
@@ -630,27 +650,20 @@
 class W_CPPConstructorOverload(W_CPPOverload):
     _attrs_ = []
 
-    @unwrap_spec(args_w='args_w')
-    def descr_get(self, w_cppinstance, args_w):
-        if self.space.is_w(w_cppinstance, self.space.w_None):
-            return self  # unbound (TODO: probably useless)
-        cppol = W_CPPConstructorOverload(self.space, self.scope, self.functions, self.flags)
-        cppol.w_this = w_cppinstance
-        return cppol     # bound
+    def __init__(self, space, decl_scope, funcs, flags = OVERLOAD_FLAGS_USE_FFI):
+        W_CPPOverload.__init__(self, space, decl_scope, funcs, flags)
+        self.flags &= ~OVERLOAD_FLAGS_USE_FFI
 
     @unwrap_spec(args_w='args_w')
-    def call(self, args_w):
+    def call_args(self, args_w):
+        jit.promote(self)
         # TODO: factor out the following:
         if capi.c_is_abstract(self.space, self.scope.handle):
             raise oefmt(self.space.w_TypeError,
                         "cannot instantiate abstract class '%s'",
                         self.scope.name)
-        if self.space.is_w(self.w_this, self.space.w_None) and len(args_w):
-            cppinstance = self.space.interp_w(W_CPPInstance, args_w[0])
-            args_w = args_w[1:]
-        else:
-            cppinstance = self.space.interp_w(W_CPPInstance, self.w_this)
-        w_result = self.call_impl(rffi.cast(capi.C_OBJECT, self.scope.handle), args_w)
+        cppinstance = self.space.interp_w(W_CPPInstance, args_w[0])
+        w_result = self.call_impl(rffi.cast(capi.C_OBJECT, self.scope.handle), args_w[1:])
         newthis = rffi.cast(capi.C_OBJECT, self.space.uint_w(w_result))
         if cppinstance is not None:
             cppinstance._rawobject = newthis
@@ -662,7 +675,7 @@
 W_CPPConstructorOverload.typedef = TypeDef(
     'CPPConstructorOverload',
     __get__    = interp2app(W_CPPConstructorOverload.descr_get),
-    __call__   = interp2app(W_CPPConstructorOverload.call),
+    __call__   = interp2app(W_CPPConstructorOverload.call_args),
     __doc__    = GetSetProperty(W_CPPConstructorOverload.fget_doc)
 )
 
@@ -725,7 +738,9 @@
         # try to match with run-time instantiations
         for cppol in self.master.overloads.values():
             try:
-                return cppol.descr_get(self.w_this, []).call(args_w)
+                if not self.space.is_w(self.w_this, self.space.w_None):
+                    return self.space.call_obj_args(cppol, self.w_this, Arguments(self.space, args_w))
+                return self.space.call_args(cppol, Arguments(self.space, args_w))
             except Exception:
                 pass    # completely ignore for now; have to see whether errors become confusing
 
@@ -748,7 +763,9 @@
             except KeyError:
                 self.master.overloads[fullname] = method
 
-        return method.descr_get(self.w_this, []).call(args_w)
+        if not self.space.is_w(self.w_this, self.space.w_None):
+	    return self.space.call_obj_args(method, self.w_this, Arguments(self.space, args_w))
+        return self.space.call_args(method, Arguments(self.space, args_w))
 
     def getitem_impl(self, name, args_w):
         space = self.space
@@ -771,25 +788,25 @@
             if c_fullname != fullname:
                 self.master.overloads[c_fullname] = method
 
-        return method.descr_get(self.w_this, [])
+        return method.descr_get(self.w_this, None)
 
 
 class W_CPPTemplateOverload(W_CPPOverload, TemplateOverloadMixin):
     """App-level dispatcher to allow both lookup/instantiation of templated methods and
     dispatch among overloads between templated and non-templated method."""
 
-    _attrs_ = ['name', 'overloads', 'master']
+    _attrs_ = ['name', 'overloads', 'master', 'w_this']
     _immutable_fields_ = ['name']
 
-    def __init__(self, space, name, declaring_scope, functions, flags = OVERLOAD_FLAGS_USE_FFI):
-         W_CPPOverload.__init__(self, space, declaring_scope, functions, flags)
+    def __init__(self, space, name, decl_scope, functions, flags = OVERLOAD_FLAGS_USE_FFI):
+         W_CPPOverload.__init__(self, space, decl_scope, functions, flags)
          self.name = name
          self.overloads = {}
          self.master = self
+	 self.w_this = space.w_None
 
-    @unwrap_spec(args_w='args_w')
-    def descr_get(self, w_cppinstance, args_w):
-        # like W_CPPOverload, but returns W_CPPTemplateOverload
+    def descr_get(self, w_cppinstance, w_cls=None):
+	# TODO: don't return copy, but bind in an external object (like W_CPPOverload)
         if self.space.is_w(w_cppinstance, self.space.w_None):
             return self  # unbound, so no new instance needed
         cppol = W_CPPTemplateOverload(self.space, self.name, self.scope, self.functions, self.flags)
@@ -798,13 +815,13 @@
         return cppol     # bound
 
     @unwrap_spec(args_w='args_w')
-    def call(self, args_w):
+    def call_args(self, args_w):
         # direct call: either pick non-templated overload or attempt to deduce
         # the template instantiation from the argument types
 
         # try existing overloads or compile-time instantiations
         try:
-            return W_CPPOverload.call(self, args_w)
+            return W_CPPOverload.call_args(self, args_w)
         except Exception:
             pass
 
@@ -814,6 +831,9 @@
     def getitem(self, args_w):
         return self.getitem_impl(self.name, args_w)
 
+    def getname(self, space):
+        return self.name.decode('latin-1')
+
     def __repr__(self):
         return "W_CPPTemplateOverload(%s)" % [f.prototype() for f in self.functions]
 
@@ -821,7 +841,7 @@
     'CPPTemplateOverload',
     __get__     = interp2app(W_CPPTemplateOverload.descr_get),
     __getitem__ = interp2app(W_CPPTemplateOverload.getitem),
-    __call__    = interp2app(W_CPPTemplateOverload.call),
+    __call__    = interp2app(W_CPPTemplateOverload.call_args),
     __useffi__  = GetSetProperty(W_CPPTemplateOverload.fget_useffi, W_CPPTemplateOverload.fset_useffi),
     __doc__     = GetSetProperty(W_CPPTemplateOverload.fget_doc)
 )
@@ -830,18 +850,18 @@
     """App-level dispatcher to allow both lookup/instantiation of templated methods and
     dispatch among overloads between templated and non-templated method."""
 
-    _attrs_ = ['name', 'overloads', 'master']
+    _attrs_ = ['name', 'overloads', 'master', 'w_this']
     _immutable_fields_ = ['name']
 
-    def __init__(self, space, name, declaring_scope, functions, flags = OVERLOAD_FLAGS_USE_FFI):
-         W_CPPStaticOverload.__init__(self, space, declaring_scope, functions, flags)
+    def __init__(self, space, name, decl_scope, funcs, flags = OVERLOAD_FLAGS_USE_FFI):
+         W_CPPStaticOverload.__init__(self, space, decl_scope, funcs, flags)
          self.name = name
          self.overloads = {}
          self.master = self
+	 self.w_this = space.w_None
 
-    @unwrap_spec(args_w='args_w')
-    def descr_get(self, w_cppinstance, args_w):
-        # like W_CPPStaticOverload, but returns W_CPPTemplateStaticOverload
+    def descr_get(self, w_cppinstance, w_cls=None):
+	# TODO: don't return copy, but bind in an external object (like W_CPPOverload)
         if isinstance(w_cppinstance, W_CPPInstance):
             cppinstance = self.space.interp_w(W_CPPInstance, w_cppinstance)
             if cppinstance.clsdecl.handle != self.scope.handle:
@@ -852,13 +872,13 @@
         return self      # unbound
 
     @unwrap_spec(args_w='args_w')
-    def call(self, args_w):
+    def call_args(self, args_w):
         # direct call: either pick non-templated overload or attempt to deduce
         # the template instantiation from the argument types
 
         # try existing overloads or compile-time instantiations
         try:
-            return W_CPPStaticOverload.call(self, args_w)
+            return W_CPPStaticOverload.call_args(self, args_w)
         except Exception:
             pass
 
@@ -869,6 +889,9 @@
     def getitem(self, args_w):
         return self.getitem_impl(self.name, args_w)
 
+    def getname(self, space):
+        return self.name.decode('latin-1')
+
     def __repr__(self):
         return "W_CPPTemplateStaticOverload(%s)" % [f.prototype() for f in self.functions]
 
@@ -876,7 +899,7 @@
     'CPPTemplateStaticOverload',
     __get__     = interp2app(W_CPPTemplateStaticOverload.descr_get),
     __getitem__ = interp2app(W_CPPTemplateStaticOverload.getitem),
-    __call__    = interp2app(W_CPPTemplateStaticOverload.call),
+    __call__    = interp2app(W_CPPTemplateStaticOverload.call_args),
     __useffi__  = GetSetProperty(W_CPPTemplateStaticOverload.fget_useffi, W_CPPTemplateStaticOverload.fset_useffi),
     __doc__     = GetSetProperty(W_CPPTemplateStaticOverload.fget_doc)
 )
@@ -898,9 +921,9 @@
     _attrs_ = ['space', 'scope', 'converter', 'offset']
     _immutable_fields = ['scope', 'converter', 'offset']
 
-    def __init__(self, space, declaring_scope, type_name, offset):
+    def __init__(self, space, decl_scope, type_name, offset):
         self.space = space
-        self.scope = declaring_scope
+        self.scope = decl_scope
         self.converter = converter.get_converter(self.space, type_name, '')
         self.offset = offset
 
@@ -1417,7 +1440,7 @@
                     ol = W_CPPStaticOverload(self.space, nss, funcs[:])
                     # TODO: cache this operator (not done yet, as the above does not
                     # select all overloads)
-                    return ol.call([self, w_other])
+                    return ol.call_args([self, w_other])
         except OperationError as e:
             if not e.match(self.space, self.space.w_TypeError):
                 raise
diff --git a/pypy/module/_cppyy/test/test_zjit.py b/pypy/module/_cppyy/test/test_zjit.py
--- a/pypy/module/_cppyy/test/test_zjit.py
+++ b/pypy/module/_cppyy/test/test_zjit.py
@@ -205,6 +205,9 @@
     def exception_match(self, typ, sub):
         return typ is sub
 
+    def is_none(self, w_obj):
+        return w_obj is None
+
     def is_w(self, w_one, w_two):
         return w_one is w_two
 
@@ -268,6 +271,9 @@
     def call_function(self, w_func, *args_w):
         return None
 
+    def call_obj_args(self, w_callable, w_obj, args):
+        return w_callable.call_args([w_obj]+args)
+
     def _freeze_(self):
         return True
 
@@ -283,19 +289,19 @@
         def f():
             cls  = interp_cppyy.scope_byname(space, "example01")
             inst = interp_cppyy._bind_object(space, FakeInt(0), cls, True)
-            cls.get_overload("__init__").descr_get(inst, []).call([FakeInt(0)])
+            cls.get_overload("__init__").descr_get(inst, []).call_args([FakeInt(0)])
             cppmethod = cls.get_overload(method_name)
             assert isinstance(inst, interp_cppyy.W_CPPInstance)
             i = 10
             while i > 0:
                 drv.jit_merge_point(inst=inst, cppmethod=cppmethod, i=i)
-                cppmethod.descr_get(inst, []).call([FakeInt(i)])
+                cppmethod.descr_get(inst, []).call_args([FakeInt(i)])
                 i -= 1
             return 7
         f()
         space = FakeSpace()
         result = self.meta_interp(f, [], listops=True, backendopt=True, listcomp=True)
-        self.check_jitcell_token_count(0)   # same for fast and slow path??
+        self.check_jitcell_token_count(1)
         # rely on replacement of capi calls to raise exception instead (see FakeSpace.__init__)
 
     @py.test.mark.dont_track_allocations("cppmethod.cif_descr kept 'leaks'")
diff --git a/pypy/module/sys/test/test_sysmodule.py b/pypy/module/sys/test/test_sysmodule.py
--- a/pypy/module/sys/test/test_sysmodule.py
+++ b/pypy/module/sys/test/test_sysmodule.py
@@ -358,6 +358,29 @@
             print(ascii(err.getvalue()))
             assert err.getvalue().endswith("ValueError: %s\n" % input)
 
+    def test_excepthook_flushes_stdout(self): r"""
+        import sys, io
+        savestdout = sys.stdout
+        out = io.StringIO()
+        sys.stdout = out
+
+        eh = sys.__excepthook__
+
+        try:
+            raise ValueError(42)
+        except ValueError as exc:
+            print("hello")     # with end-of-line
+            eh(*sys.exc_info())
+        try:
+            raise ValueError(42)
+        except ValueError as exc:
+            print(123, 456, end="")     # no end-of-line here
+            eh(*sys.exc_info())
+
+        sys.stdout = savestdout
+        assert out.getvalue() == 'hello\n123 456'   # no final \n added in 3.x
+        """
+
     # FIXME: testing the code for a lost or replaced excepthook in
     # Python/pythonrun.c::PyErr_PrintEx() is tricky.
 
diff --git a/pypy/objspace/std/intobject.py b/pypy/objspace/std/intobject.py
--- a/pypy/objspace/std/intobject.py
+++ b/pypy/objspace/std/intobject.py
@@ -371,20 +371,23 @@
     return wrapint(space, a)
 
 
- at jit.look_inside_iff(lambda space, iv, iw, iz:
-                     jit.isconstant(iw) and jit.isconstant(iz))
 def _pow(space, iv, iw, iz):
     """Helper for pow"""
-    if iw < 0:
-        if iz != 0:
-            raise oefmt(space.w_ValueError,
-                        "pow() 2nd argument cannot be negative when 3rd "
-                        "argument specified")
+    if iz == 0:
+        return _pow_nomod(iv, iw)
+    else:
+        return _pow_mod(space, iv, iw, iz)
+
+ at jit.look_inside_iff(lambda iv, iw: jit.isconstant(iw))
+def _pow_nomod(iv, iw):
+    if iw <= 0:
+        if iw == 0:
+            return 1
         # bounce it, since it always returns float
         raise ValueError
     temp = iv
     ix = 1
-    while iw > 0:
+    while True:
         if iw & 1:
             try:
                 ix = ovfcheck(ix * temp)
@@ -397,12 +400,40 @@
             temp = ovfcheck(temp * temp) # Square the value of temp
         except OverflowError:
             raise
-        if iz:
-            # If we did a multiplication, perform a modulo
-            ix %= iz
-            temp %= iz
-    if iz:
-        ix %= iz
+    return ix
+
+ at jit.look_inside_iff(lambda space, iv, iw, iz:
+                     jit.isconstant(iw) and jit.isconstant(iz))
+def _pow_mod(space, iv, iw, iz):
+    from rpython.rlib.rarithmetic import mulmod
+
+    if iw <= 0:
+        if iw == 0:
+            return 1 % iz   # != 1, for iz == 1 or iz < 0
+        raise oefmt(space.w_ValueError,
+                    "pow() 2nd argument cannot be negative when 3rd "
+                    "argument specified")
+    if iz < 0:
+        try:
+            iz = ovfcheck(-iz)
+        except OverflowError:
+            raise
+        iz_negative = True
+    else:
+        iz_negative = False
+
+    temp = iv
+    ix = 1
+    while True:
+        if iw & 1:
+            ix = mulmod(ix, temp, iz)
+        iw >>= 1   # Shift exponent down by 1 bit
+        if iw == 0:
+            break
+        temp = mulmod(temp, temp, iz)
+
+    if iz_negative and ix > 0:
+        ix -= iz
     return ix
 
 
diff --git a/pypy/objspace/std/test/test_intobject.py b/pypy/objspace/std/test/test_intobject.py
--- a/pypy/objspace/std/test/test_intobject.py
+++ b/pypy/objspace/std/test/test_intobject.py
@@ -1,7 +1,8 @@
 # encoding: utf-8
 import sys
+from pypy.interpreter.error import OperationError
 from pypy.objspace.std import intobject as iobj
-from rpython.rlib.rarithmetic import r_uint, is_valid_int
+from rpython.rlib.rarithmetic import r_uint, is_valid_int, intmask
 from rpython.rlib.rbigint import rbigint
 
 class TestW_IntObject:
@@ -169,6 +170,63 @@
         assert space.isinstance_w(v, space.w_int)
         assert space.bigint_w(v).eq(rbigint.fromlong(pow(10, 20)))
 
+    try:
+        from hypothesis import given, strategies, example
+    except ImportError:
+        pass
+    else:
+        @given(
+           a=strategies.integers(min_value=-sys.maxint-1, max_value=sys.maxint),
+           b=strategies.integers(min_value=-sys.maxint-1, max_value=sys.maxint),
+           c=strategies.integers(min_value=-sys.maxint-1, max_value=sys.maxint))
+        @example(0, 0, -sys.maxint-1)
+        @example(0, 1, -sys.maxint-1)
+        @example(1, 0, -sys.maxint-1)
+        def test_hypot_pow(self, a, b, c):
+            if c == 0:
+                return
+            #
+            # "pow(a, b, c)": if b < 0, should get an app-level ValueError.
+            # Otherwise, should always work except if c == -maxint-1
+            if b < 0:
+                expected = "app-level ValueError"
+            elif b > 0 and c == -sys.maxint-1:
+                expected = OverflowError
+            else:
+                expected = pow(a, b, c)
+
+            try:
+                result = iobj._pow(self.space, a, b, c)
+            except OperationError as e:
+                assert ('ValueError: pow() 2nd argument cannot be negative '
+                        'when 3rd argument specified' == e.errorstr(self.space))
+                result = "app-level ValueError"
+            except OverflowError:
+                result = OverflowError
+            assert result == expected
+
+        @given(
+           a=strategies.integers(min_value=-sys.maxint-1, max_value=sys.maxint),
+           b=strategies.integers(min_value=-sys.maxint-1, max_value=sys.maxint))
+        def test_hypot_pow_nomod(self, a, b):
+            # "a ** b": detect overflows and ValueErrors
+            if b < 0:
+                expected = ValueError
+            elif b > 128 and not (-1 <= a <= 1):
+                expected = OverflowError
+            else:
+                expected = a ** b
+                if expected != intmask(expected):
+                    expected = OverflowError
+
+            try:
+                result = iobj._pow(self.space, a, b, 0)
+            except ValueError:
+                result = ValueError
+            except OverflowError:
+                result = OverflowError
+            assert result == expected
+
     def test_neg(self):
         space = self.space
         x = 42
diff --git a/rpython/rlib/rarithmetic.py b/rpython/rlib/rarithmetic.py
--- a/rpython/rlib/rarithmetic.py
+++ b/rpython/rlib/rarithmetic.py
@@ -840,6 +840,31 @@
         return z
 
 
+ at specialize.memo()
+def check_support_int128():
+    from rpython.rtyper.lltypesystem import rffi
+    return hasattr(rffi, '__INT128_T')
+
+def mulmod(a, b, c):
+    """Computes (a * b) % c.
+    Assumes c > 0, and returns a nonnegative result.
+    """
+    assert c > 0
+    if LONG_BIT < LONGLONG_BIT:
+        a = r_longlong(a)
+        b = r_longlong(b)
+        return intmask((a * b) % c)
+    elif check_support_int128():
+        a = r_longlonglong(a)
+        b = r_longlonglong(b)
+        return intmask((a * b) % c)
+    else:
+        from rpython.rlib.rbigint import rbigint
+        a = rbigint.fromlong(a)
+        b = rbigint.fromlong(b)
+        return a.mul(b).int_mod(c).toint()
+
+
 # String parsing support
 # ---------------------------
 
diff --git a/rpython/rlib/rbigint.py b/rpython/rlib/rbigint.py
--- a/rpython/rlib/rbigint.py
+++ b/rpython/rlib/rbigint.py
@@ -1,6 +1,7 @@
 from rpython.rlib.rarithmetic import LONG_BIT, intmask, longlongmask, r_uint, r_ulonglong
 from rpython.rlib.rarithmetic import ovfcheck, r_longlong, widen
 from rpython.rlib.rarithmetic import most_neg_value_of_same_type
+from rpython.rlib.rarithmetic import check_support_int128
 from rpython.rlib.rstring import StringBuilder
 from rpython.rlib.debug import make_sure_not_resized, check_regular_int
 from rpython.rlib.objectmodel import we_are_translated, specialize, not_rpython
@@ -10,7 +11,7 @@
 
 import math, sys
 
-SUPPORT_INT128 = hasattr(rffi, '__INT128_T')
+SUPPORT_INT128 = check_support_int128()
 BYTEORDER = sys.byteorder
 
 # note about digit sizes:
diff --git a/rpython/rlib/rvmprof/dummy.py b/rpython/rlib/rvmprof/dummy.py
--- a/rpython/rlib/rvmprof/dummy.py
+++ b/rpython/rlib/rvmprof/dummy.py
@@ -23,4 +23,4 @@
         pass
 
     def stop_sampling(self):
-        pass
+        return -1
diff --git a/rpython/rlib/test/test_rarithmetic.py b/rpython/rlib/test/test_rarithmetic.py
--- a/rpython/rlib/test/test_rarithmetic.py
+++ b/rpython/rlib/test/test_rarithmetic.py
@@ -1,5 +1,6 @@
 from rpython.rtyper.test.tool import BaseRtypingTest
 from rpython.rtyper.test.test_llinterp import interpret
+from rpython.rlib import rarithmetic
 from rpython.rlib.rarithmetic import *
 from rpython.rlib.rstring import ParseStringError, ParseStringOverflowError
 from hypothesis import given, strategies, assume
@@ -731,3 +732,17 @@
     py.test.raises(OverflowError, ovfcheck_int32_sub, 2**30, -2**30)
     assert ovfcheck_int32_mul(-2**16, 2**15) == -2**31
     py.test.raises(OverflowError, ovfcheck_int32_mul, -2**16, -2**15)
+
+ at given(strategies.integers(min_value=-sys.maxint-1, max_value=sys.maxint),
+       strategies.integers(min_value=-sys.maxint-1, max_value=sys.maxint),
+       strategies.integers(min_value=1, max_value=sys.maxint))
+def test_mulmod(a, b, c):
+    assert mulmod(a, b, c) == (a * b) % c
+    #
+    import rpython.rlib.rbigint  # import before patching check_support_int128
+    prev = rarithmetic.check_support_int128
+    try:
+        rarithmetic.check_support_int128 = lambda: False
+        assert mulmod(a, b, c) == (a * b) % c
+    finally:
+        rarithmetic.check_support_int128 = prev
diff --git a/rpython/translator/c/test/test_typed.py b/rpython/translator/c/test/test_typed.py
--- a/rpython/translator/c/test/test_typed.py
+++ b/rpython/translator/c/test/test_typed.py
@@ -962,3 +962,12 @@
         f = self.getcompiled(func, [int])
         res = f(2)
         assert res == 1     # and not 2
+
+    def test_mulmod(self):
+        from rpython.rlib.rarithmetic import mulmod
+
+        def func(a, b, c):
+            return mulmod(a, b, c)
+        f = self.getcompiled(func, [int, int, int])
+        res = f(1192871273, 1837632879, 2001286281)
+        assert res == 1573897320

From pypy.commits at gmail.com  Sun Jul  8 12:11:35 2018
From: pypy.commits at gmail.com (cfbolz)
Date: Sun, 08 Jul 2018 09:11:35 -0700 (PDT)
Subject: [pypy-commit] pypy default: because I felt like it: add some
 missing people and verbs to makecontributor.py
Message-ID: <5b4237b7.1c69fb81.21a2.3d55@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: 
Changeset: r94828:c520c8e7a603
Date: 2018-07-08 18:10 +0200
http://bitbucket.org/pypy/pypy/changeset/c520c8e7a603/

Log:	because I felt like it: add some missing people and verbs to
	makecontributor.py

diff --git a/pypy/doc/tool/makecontributor.py b/pypy/doc/tool/makecontributor.py
--- a/pypy/doc/tool/makecontributor.py
+++ b/pypy/doc/tool/makecontributor.py
@@ -18,12 +18,13 @@
     'Antonio Cuni': ['antocuni', 'anto'],
     'Armin Rigo': ['arigo', 'arfigo', 'armin', 'arigato'],
     'Maciej Fijalkowski': ['fijal'],
-    'Carl Friedrich Bolz-Tereick': ['Carl Friedrich Bolz', 'cfbolz', 'cf'],
+    'Carl Friedrich Bolz-Tereick': ['Carl Friedrich Bolz', 'cfbolz', 'cf', 'cbolz'],
     'Samuele Pedroni': ['pedronis', 'samuele', 'samule'],
-    'Richard Plangger':['planrich'],
-    'Michael Hudson': ['mwh'],
+    'Richard Plangger': ['planrich', 'plan_rich'],
+    'Remi Meier': ['remi'],
+    'Michael Hudson-Doyle': ['mwh', 'Michael Hudson'],
     'Holger Krekel': ['hpk', 'holger krekel', 'holger', 'hufpk'],
-    "Amaury Forgeot d'Arc": ['afa'],
+    "Amaury Forgeot d'Arc": ['afa', 'amauryfa at gmail.com'],
     'Alex Gaynor': ['alex', 'agaynor'],
     'David Schneider': ['bivab', 'david'],
     'Christian Tismer': ['chris', 'christian', 'tismer',
@@ -41,7 +42,7 @@
     'Mark Pearse': ['mwp'],
     'Toon Verwaest': ['tverwaes'],
     'Eric van Riet Paap': ['ericvrp'],
-    'Jacob Hallen': ['jacob', 'jakob'],
+    'Jacob Hallen': ['jacob', 'jakob', 'jacob hallen'],
     'Anders Lehmann': ['ale', 'anders'],
     'Bert Freudenberg': ['bert'],
     'Boris Feigin': ['boris', 'boria'],
@@ -69,19 +70,25 @@
     'Manuel Jacob': ['mjacob'],
     'Rami Chowdhury': ['necaris'],
     'Stanislaw Halik': ['Stanislaw Halik', 'w31rd0'],
-    'Wenzhu Man':['wenzhu man', 'wenzhuman'],
-    'Anton Gulenko':['anton gulenko', 'anton_gulenko'],
-    'Richard Lancaster':['richardlancaster'],
-    'William Leslie':['William ML Leslie'],
-    'Spenser Bauman':['Spenser Andrew Bauman'],
-    'Raffael Tfirst':['raffael.tfirst at gmail.com'],
-    'timo':['timo at eistee.fritz.box'],
-    'Jasper Schulz':['Jasper.Schulz', 'jbs'],
-    'Aaron Gallagher':['"Aaron Gallagher'],
-    'Yasir Suhail':['yasirs'],
+    'Wenzhu Man': ['wenzhu man', 'wenzhuman'],
+    'Anton Gulenko': ['anton gulenko', 'anton_gulenko'],
+    'Richard Lancaster': ['richardlancaster'],
+    'William Leslie': ['William ML Leslie'],
+    'Spenser Bauman': ['Spenser Andrew Bauman'],
+    'Raffael Tfirst': ['raffael.tfirst at gmail.com'],
+    'timo': ['timo at eistee.fritz.box'],
+    'Jasper Schulz': ['Jasper.Schulz', 'jbs'],
+    'Aaron Gallagher': ['"Aaron Gallagher'],
+    'Yasir Suhail': ['yasirs'],
     'Squeaky': ['squeaky'],
-    "Amaury Forgeot d'Arc": ['amauryfa at gmail.com'],
     "Dodan Mihai": ['mihai.dodan at gmail.com'],
+    'Wim Lavrijsen': ['wlav'],
+    'Toon Verwaest': ['toon', 'tverwaes'],
+    'Seo Sanghyeon': ['sanxiyn'],
+    'Leonardo Santagada': ['santagada'],
+    'Laurence Tratt': ['ltratt'],
+    'Pieter Zieschang': ['pzieschang', 'p_zieschang at yahoo.de'],
+    'John Witulski': ['witulski'],
     }
 
 alias_map = {}
@@ -103,7 +110,8 @@
         return set()
     ignore_words = ['around', 'consulting', 'yesterday', 'for a bit', 'thanks',
                     'in-progress', 'bits of', 'even a little', 'floating',
-                    'a bit', 'reviewing']
+                    'a bit', 'reviewing', 'looking', 'advising', 'partly', 'ish',
+                    'watching', 'mostly', 'jumping']
     sep_words = ['and', ';', '+', '/', 'with special  by']
     nicknames = match.group(1)
     for word in ignore_words:

From pypy.commits at gmail.com  Sun Jul  8 14:28:53 2018
From: pypy.commits at gmail.com (mattip)
Date: Sun, 08 Jul 2018 11:28:53 -0700 (PDT)
Subject: [pypy-commit] pypy default: "test" for tzinfo dealloc (leakchecker
 is flaky, so use print + pytest ... -s)
Message-ID: <5b4257e5.1c69fb81.4d727.155b@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r94829:19fdbdf52ca1
Date: 2018-07-08 11:24 -0700
http://bitbucket.org/pypy/pypy/changeset/19fdbdf52ca1/

Log:	"test" for tzinfo dealloc (leakchecker is flaky, so use print +
	pytest ... -s)

diff --git a/pypy/module/cpyext/test/test_datetime.py b/pypy/module/cpyext/test/test_datetime.py
--- a/pypy/module/cpyext/test/test_datetime.py
+++ b/pypy/module/cpyext/test/test_datetime.py
@@ -307,18 +307,20 @@
         from datetime import tzinfo, datetime, timedelta, time
         # copied from datetime documentation
         class GMT1(tzinfo):
-           def utcoffset(self, dt):
-               return timedelta(hours=1) + self.dst(dt)
-           def dst(self, dt):
-               return timedelta(0)
-           def tzname(self,dt):
+            def __del__(self):
+                print 'deleting GMT1'
+            def utcoffset(self, dt):
+                return timedelta(hours=1) + self.dst(dt)
+            def dst(self, dt):
+                return timedelta(0)
+            def tzname(self,dt):
                 return "GMT +1"
         gmt1 = GMT1()
         dt1 = module.time_with_tzinfo(gmt1)
         assert dt1 == time(6, 6, 6, 6, gmt1)
         assert '+01' in str(dt1)
-        assert module.datetime_with_tzinfo(gmt1) == datetime(
-            2000, 6, 6, 6, 6, 6, 6, gmt1)
+        dt_tz = module.datetime_with_tzinfo(gmt1)
+        assert dt_tz == datetime(2000, 6, 6, 6, 6, 6, 6, gmt1)
 
     def test_checks(self):
         module = self.import_extension('foo', [

From pypy.commits at gmail.com  Sun Jul  8 14:28:55 2018
From: pypy.commits at gmail.com (mattip)
Date: Sun, 08 Jul 2018 11:28:55 -0700 (PDT)
Subject: [pypy-commit] pypy default: cleanup type_alloc, type_dealloc,
 fixes #2855
Message-ID: <5b4257e7.1c69fb81.c496f.0ee2@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r94830:6239e5f4d6f1
Date: 2018-07-08 11:27 -0700
http://bitbucket.org/pypy/pypy/changeset/6239e5f4d6f1/

Log:	cleanup type_alloc, type_dealloc, fixes #2855

diff --git a/pypy/module/cpyext/cdatetime.py b/pypy/module/cpyext/cdatetime.py
--- a/pypy/module/cpyext/cdatetime.py
+++ b/pypy/module/cpyext/cdatetime.py
@@ -120,21 +120,10 @@
                    dealloc=type_dealloc,
                   )
 
-    # why do we need date_dealloc? Since W_DateTime_Date is the base class for
-    # app level datetime.date. If a c-extension class uses datetime.date for its
-    # base class and defines a tp_dealloc, we will get this:
-    # c_class->tp_dealloc == tp_dealloc_func
-    # c_class->tp_base == datetime.date, 
-    #                     datetime.date->tp_dealloc = _PyPy_subtype_dealloc
-    # datetime.date->tp_base = W_DateTime_Date
-    #                    W_DateTime_Date->tp_dealloc = _PyPy_subtype_dealloc
-    # but _PyPy_subtype_dealloc will call tp_dealloc_func, which can call its
-    # base's tp_dealloc and we get recursion. So break the recursion by setting
-    # W_DateTime_Date->tp_dealloc
     make_typedescr(W_DateTime_Date.typedef,
                    basestruct=PyDateTime_DateTime.TO,
                    attach=type_attach,
-                   dealloc=date_dealloc,
+                   dealloc=type_dealloc,
                   )
 
     make_typedescr(W_DateTime_Delta.typedef,
@@ -144,30 +133,39 @@
 
 def type_attach(space, py_obj, w_obj, w_userdata=None):
     '''Fills a newly allocated py_obj from the w_obj
+    If it is a datetime.time or datetime.datetime, it may have tzinfo
     '''
-    if space.type(w_obj).name == 'date':
-        # No tzinfo
-        return
-    py_datetime = rffi.cast(PyDateTime_Time, py_obj)
-    w_tzinfo = space.getattr(w_obj, space.newtext('tzinfo'))
-    if space.is_none(w_tzinfo):
-        py_datetime.c_hastzinfo = cts.cast('unsigned char', 0)
-        py_datetime.c_tzinfo = lltype.nullptr(PyObject.TO)
-    else:
-        py_datetime.c_hastzinfo = cts.cast('unsigned char', 1)
-        py_datetime.c_tzinfo = make_ref(space, w_tzinfo)
+    if datetimeAPI_global[0].c_TimeType == py_obj.c_ob_type:
+        py_datetime = rffi.cast(PyDateTime_Time, py_obj)
+        w_tzinfo = space.getattr(w_obj, space.newtext('tzinfo'))
+        if space.is_none(w_tzinfo):
+            py_datetime.c_hastzinfo = cts.cast('unsigned char', 0)
+            py_datetime.c_tzinfo = lltype.nullptr(PyObject.TO)
+        else:
+            py_datetime.c_hastzinfo = cts.cast('unsigned char', 1)
+            py_datetime.c_tzinfo = make_ref(space, w_tzinfo)
+    elif datetimeAPI_global[0].c_DateTimeType == py_obj.c_ob_type:
+        # For now this is exactly the same structure as PyDateTime_Time
+        py_datetime = rffi.cast(PyDateTime_DateTime, py_obj)
+        w_tzinfo = space.getattr(w_obj, space.newtext('tzinfo'))
+        if space.is_none(w_tzinfo):
+            py_datetime.c_hastzinfo = cts.cast('unsigned char', 0)
+            py_datetime.c_tzinfo = lltype.nullptr(PyObject.TO)
+        else:
+            py_datetime.c_hastzinfo = cts.cast('unsigned char', 1)
+            py_datetime.c_tzinfo = make_ref(space, w_tzinfo)
 
 @slot_function([PyObject], lltype.Void)
 def type_dealloc(space, py_obj):
-    py_datetime = rffi.cast(PyDateTime_Time, py_obj)
-    if (widen(py_datetime.c_hastzinfo) != 0):
-        decref(space, py_datetime.c_tzinfo)
     from pypy.module.cpyext.object import _dealloc
-    _dealloc(space, py_obj)
-
- at slot_function([PyObject], lltype.Void)
-def date_dealloc(space, py_obj):
-    from pypy.module.cpyext.object import _dealloc
+    if datetimeAPI_global[0].c_TimeType == py_obj.c_ob_type:
+        py_datetime = rffi.cast(PyDateTime_Time, py_obj)
+        if (widen(py_datetime.c_hastzinfo) != 0):
+            decref(space, py_datetime.c_tzinfo)
+    elif datetimeAPI_global[0].c_DateTimeType == py_obj.c_ob_type:
+        py_datetime = rffi.cast(PyDateTime_DateTime, py_obj)
+        if (widen(py_datetime.c_hastzinfo) != 0):
+            decref(space, py_datetime.c_tzinfo)
     _dealloc(space, py_obj)
 
 def timedeltatype_attach(space, py_obj, w_obj, w_userdata=None):

From pypy.commits at gmail.com  Sun Jul  8 14:39:21 2018
From: pypy.commits at gmail.com (mattip)
Date: Sun, 08 Jul 2018 11:39:21 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: tabs -> spaces
Message-ID: <5b425a59.1c69fb81.d9341.f3ca@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: py3.5
Changeset: r94831:75b70a7697a1
Date: 2018-07-08 11:38 -0700
http://bitbucket.org/pypy/pypy/changeset/75b70a7697a1/

Log:	tabs -> spaces

diff --git a/pypy/module/_cppyy/interp_cppyy.py b/pypy/module/_cppyy/interp_cppyy.py
--- a/pypy/module/_cppyy/interp_cppyy.py
+++ b/pypy/module/_cppyy/interp_cppyy.py
@@ -455,7 +455,7 @@
 class MethodWithProps(Method):
     # allow user to determine ffi use rules per overload
     def fget_useffi(self, space):
-	f = space.interp_w(W_CPPOverload, self.w_function)
+        f = space.interp_w(W_CPPOverload, self.w_function)
         return f.fget_useffi(space)
 
     @unwrap_spec(value=bool)
@@ -623,7 +623,7 @@
             # onto a class and w_this should be set
             cppinstance = self.space.interp_w(W_CPPInstance, w_obj)
             if cppinstance.clsdecl.handle != self.scope.handle:
-	        return MethodWithProps(self.space, self, w_obj)    # bound
+                return MethodWithProps(self.space, self, w_obj)    # bound
         return self      # unbound
 
     @unwrap_spec(args_w='args_w')
@@ -764,7 +764,7 @@
                 self.master.overloads[fullname] = method
 
         if not self.space.is_w(self.w_this, self.space.w_None):
-	    return self.space.call_obj_args(method, self.w_this, Arguments(self.space, args_w))
+            return self.space.call_obj_args(method, self.w_this, Arguments(self.space, args_w))
         return self.space.call_args(method, Arguments(self.space, args_w))
 
     def getitem_impl(self, name, args_w):
@@ -803,10 +803,10 @@
          self.name = name
          self.overloads = {}
          self.master = self
-	 self.w_this = space.w_None
+         self.w_this = space.w_None
 
     def descr_get(self, w_cppinstance, w_cls=None):
-	# TODO: don't return copy, but bind in an external object (like W_CPPOverload)
+        # TODO: don't return copy, but bind in an external object (like W_CPPOverload)
         if self.space.is_w(w_cppinstance, self.space.w_None):
             return self  # unbound, so no new instance needed
         cppol = W_CPPTemplateOverload(self.space, self.name, self.scope, self.functions, self.flags)
@@ -858,10 +858,10 @@
          self.name = name
          self.overloads = {}
          self.master = self
-	 self.w_this = space.w_None
+         self.w_this = space.w_None
 
     def descr_get(self, w_cppinstance, w_cls=None):
-	# TODO: don't return copy, but bind in an external object (like W_CPPOverload)
+        # TODO: don't return copy, but bind in an external object (like W_CPPOverload)
         if isinstance(w_cppinstance, W_CPPInstance):
             cppinstance = self.space.interp_w(W_CPPInstance, w_cppinstance)
             if cppinstance.clsdecl.handle != self.scope.handle:

From pypy.commits at gmail.com  Sun Jul  8 16:33:20 2018
From: pypy.commits at gmail.com (mattip)
Date: Sun, 08 Jul 2018 13:33:20 -0700 (PDT)
Subject: [pypy-commit] pypy unicode-utf8-py3: remove more deprecated
 space.str_w
Message-ID: <5b427510.1c69fb81.31a4b.1a86@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: unicode-utf8-py3
Changeset: r94832:383fb0dd6744
Date: 2018-07-08 12:18 -0700
http://bitbucket.org/pypy/pypy/changeset/383fb0dd6744/

Log:	remove more deprecated space.str_w

diff --git a/pypy/interpreter/astcompiler/test/test_compiler.py b/pypy/interpreter/astcompiler/test/test_compiler.py
--- a/pypy/interpreter/astcompiler/test/test_compiler.py
+++ b/pypy/interpreter/astcompiler/test/test_compiler.py
@@ -76,7 +76,7 @@
         space = self.space
         pyco_expr = PyCode._from_code(space, co_expr)
         w_res = pyco_expr.exec_host_bytecode(w_dict, w_dict)
-        res = space.str_w(space.repr(w_res))
+        res = space.text_w(space.repr(w_res))
         expected_repr = self.get_py3_repr(expected)
         if isinstance(expected, float):
             # Float representation can vary a bit between interpreter
@@ -1425,7 +1425,7 @@
             ''', d)
             return d['f'](5)
         """)
-        assert 'generator' in space.str_w(space.repr(w_generator))
+        assert 'generator' in space.text_w(space.repr(w_generator))
 
     def test_folding_of_list_constants(self):
         for source in (
diff --git a/pypy/interpreter/pyparser/test/test_parsestring.py b/pypy/interpreter/pyparser/test/test_parsestring.py
--- a/pypy/interpreter/pyparser/test/test_parsestring.py
+++ b/pypy/interpreter/pyparser/test/test_parsestring.py
@@ -112,14 +112,14 @@
         space = self.space
         s = '"""' + '\\' + '\n"""'
         w_ret = parsestring.parsestr(space, None, s)
-        assert space.str_w(w_ret) == ''
+        assert space.text_w(w_ret) == ''
 
     def test_bug1(self):
         space = self.space
         expected = ['x', ' ', chr(0xc3), chr(0xa9), ' ', '\n']
         input = ["'", 'x', ' ', chr(0xc3), chr(0xa9), ' ', chr(92), 'n', "'"]
         w_ret = parsestring.parsestr(space, 'utf8', ''.join(input))
-        assert space.str_w(w_ret) == ''.join(expected)
+        assert space.text_w(w_ret) == ''.join(expected)
 
     def test_wide_unicode_in_source(self):
         if sys.maxunicode == 65535:

From pypy.commits at gmail.com  Sun Jul  8 16:33:22 2018
From: pypy.commits at gmail.com (mattip)
Date: Sun, 08 Jul 2018 13:33:22 -0700 (PDT)
Subject: [pypy-commit] pypy default: add generated option templates
Message-ID: <5b427512.1c69fb81.60f41.bf3a@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r94833:1251e9da7b5f
Date: 2018-07-08 13:10 -0700
http://bitbucket.org/pypy/pypy/changeset/1251e9da7b5f/

Log:	add generated option templates

diff --git a/pypy/doc/config/objspace.disable_entrypoints.txt b/pypy/doc/config/objspace.disable_entrypoints.txt
new file mode 100644
diff --git a/pypy/doc/config/objspace.fstrings.txt b/pypy/doc/config/objspace.fstrings.txt
new file mode 100644
diff --git a/pypy/doc/config/objspace.hash.txt b/pypy/doc/config/objspace.hash.txt
new file mode 100644
diff --git a/pypy/doc/config/objspace.usemodules._frozen_importlib.txt b/pypy/doc/config/objspace.usemodules._frozen_importlib.txt
new file mode 100644
diff --git a/pypy/doc/config/objspace.usemodules._jitlog.txt b/pypy/doc/config/objspace.usemodules._jitlog.txt
new file mode 100644
diff --git a/pypy/doc/config/objspace.usemodules.faulthandler.txt b/pypy/doc/config/objspace.usemodules.faulthandler.txt
new file mode 100644
diff --git a/pypy/doc/config/translation.backendopt.replace_we_are_jitted.txt b/pypy/doc/config/translation.backendopt.replace_we_are_jitted.txt
new file mode 100644
diff --git a/pypy/doc/config/translation.jit_opencoder_model.txt b/pypy/doc/config/translation.jit_opencoder_model.txt
new file mode 100644
diff --git a/pypy/doc/config/translation.keepgoing.txt b/pypy/doc/config/translation.keepgoing.txt
new file mode 100644
diff --git a/pypy/doc/config/translation.libname.txt b/pypy/doc/config/translation.libname.txt
new file mode 100644
diff --git a/pypy/doc/config/translation.lto.txt b/pypy/doc/config/translation.lto.txt
new file mode 100644
diff --git a/pypy/doc/config/translation.profoptargs.txt b/pypy/doc/config/translation.profoptargs.txt
new file mode 100644
diff --git a/pypy/doc/config/translation.reverse_debugger.txt b/pypy/doc/config/translation.reverse_debugger.txt
new file mode 100644
diff --git a/pypy/doc/config/translation.split_gc_address_space.txt b/pypy/doc/config/translation.split_gc_address_space.txt
new file mode 100644

From pypy.commits at gmail.com  Sun Jul  8 20:59:15 2018
From: pypy.commits at gmail.com (mattip)
Date: Sun, 08 Jul 2018 17:59:15 -0700 (PDT)
Subject: [pypy-commit] pypy default: fix translation
Message-ID: <5b42b363.1c69fb81.d8a94.7c1b@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r94834:b42a5efd4312
Date: 2018-07-08 17:58 -0700
http://bitbucket.org/pypy/pypy/changeset/b42a5efd4312/

Log:	fix translation

diff --git a/pypy/module/_cppyy/interp_cppyy.py b/pypy/module/_cppyy/interp_cppyy.py
--- a/pypy/module/_cppyy/interp_cppyy.py
+++ b/pypy/module/_cppyy/interp_cppyy.py
@@ -510,7 +510,7 @@
                             not space.is_w(w_obj, space.w_None) or
                             space.is_w(w_cls, space.type(space.w_None)))
         if asking_for_bound:
-            return MethodWithProps(space, self, w_obj)
+            return MethodWithProps(space, self, w_obj, w_cls)
         else:
             return self  # unbound methods don't exist in Python 3, return self
 
@@ -626,7 +626,7 @@
             # onto a class and w_this should be set
             cppinstance = self.space.interp_w(W_CPPInstance, w_obj)
             if cppinstance.clsdecl.handle != self.scope.handle:
-                return MethodWithProps(self.space, self, w_obj)    # bound
+                return MethodWithProps(self.space, self, w_obj, w_cls)    # bound
         return self      # unbound
 
     @unwrap_spec(args_w='args_w')
diff --git a/pypy/module/cpyext/cdatetime.py b/pypy/module/cpyext/cdatetime.py
--- a/pypy/module/cpyext/cdatetime.py
+++ b/pypy/module/cpyext/cdatetime.py
@@ -135,6 +135,7 @@
     '''Fills a newly allocated py_obj from the w_obj
     If it is a datetime.time or datetime.datetime, it may have tzinfo
     '''
+    assert len(datetimeAPI_global) > 0
     if datetimeAPI_global[0].c_TimeType == py_obj.c_ob_type:
         py_datetime = rffi.cast(PyDateTime_Time, py_obj)
         w_tzinfo = space.getattr(w_obj, space.newtext('tzinfo'))
@@ -158,6 +159,7 @@
 @slot_function([PyObject], lltype.Void)
 def type_dealloc(space, py_obj):
     from pypy.module.cpyext.object import _dealloc
+    assert len(datetimeAPI_global) > 0
     if datetimeAPI_global[0].c_TimeType == py_obj.c_ob_type:
         py_datetime = rffi.cast(PyDateTime_Time, py_obj)
         if (widen(py_datetime.c_hastzinfo) != 0):

From pypy.commits at gmail.com  Mon Jul  9 00:45:49 2018
From: pypy.commits at gmail.com (mattip)
Date: Sun, 08 Jul 2018 21:45:49 -0700 (PDT)
Subject: [pypy-commit] pypy unicode-utf8-py3: fixes,
 start to handle some edge cases
Message-ID: <5b42e87d.1c69fb81.f6511.8295@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: unicode-utf8-py3
Changeset: r94835:f287dec62c4e
Date: 2018-07-08 21:38 -0700
http://bitbucket.org/pypy/pypy/changeset/f287dec62c4e/

Log:	fixes, start to handle some edge cases

diff --git a/pypy/objspace/std/dictmultiobject.py b/pypy/objspace/std/dictmultiobject.py
--- a/pypy/objspace/std/dictmultiobject.py
+++ b/pypy/objspace/std/dictmultiobject.py
@@ -122,7 +122,7 @@
         if w_fill is None:
             w_fill = space.w_None
         if space.is_w(w_type, space.w_dict):
-            ulist = space.listview_unicode(w_keys)
+            ulist = space.listview_utf8(w_keys)
             if ulist is not None:
                 strategy = space.fromcache(UnicodeDictStrategy)
                 storage = strategy.get_storage_fromkeys(ulist, w_fill)
@@ -1183,21 +1183,21 @@
 
     # we should implement the same shortcuts as we do for BytesDictStrategy
 
-    ## def setitem_str(self, w_dict, key, w_value):
-    ##     assert key is not None
-    ##     self.unerase(w_dict.dstorage)[key] = w_value
+    def setitem_str(self, w_dict, key, w_value):
+        assert key is not None
+        self.unerase(w_dict.dstorage)[key] = w_value
 
-    ## def getitem(self, w_dict, w_key):
-    ##     space = self.space
-    ##     # -- This is called extremely often.  Hack for performance --
-    ##     if type(w_key) is space.StringObjectCls:
-    ##         return self.getitem_str(w_dict, w_key.unwrap(space))
-    ##     # -- End of performance hack --
-    ##     return AbstractTypedStrategy.getitem(self, w_dict, w_key)
+    def getitem(self, w_dict, w_key):
+        space = self.space
+        # -- This is called extremely often.  Hack for performance --
+        if type(w_key) is space.StringObjectCls:
+             return self.getitem_str(w_dict, w_key.unwrap(space))
+        # -- End of performance hack --
+        return AbstractTypedStrategy.getitem(self, w_dict, w_key)
 
-    ## def getitem_str(self, w_dict, key):
-    ##     assert key is not None
-    ##     return self.unerase(w_dict.dstorage).get(key, None)
+    def getitem_str(self, w_dict, key):
+        assert key is not None
+        return self.unerase(w_dict.dstorage).get(key, None)
 
     def listview_utf8(self, w_dict):
         return self.unerase(w_dict.dstorage).keys()
@@ -1208,18 +1208,26 @@
     def wrapkey(space, key):
         return space.newutf8(key, len(key))
 
-    ## @jit.look_inside_iff(lambda self, w_dict:
-    ##                      w_dict_unrolling_heuristic(w_dict))
-    ## def view_as_kwargs(self, w_dict):
-    ##     d = self.unerase(w_dict.dstorage)
-    ##     l = len(d)
-    ##     keys, values = [None] * l, [None] * l
-    ##     i = 0
-    ##     for key, val in d.iteritems():
-    ##         keys[i] = key
-    ##         values[i] = val
-    ##         i += 1
-    ##     return keys, values
+    @jit.look_inside_iff(lambda self, w_dict:
+                         w_dict_unrolling_heuristic(w_dict))
+    def view_as_kwargs(self, w_dict):
+        d = self.unerase(w_dict.dstorage)
+        l = len(d)
+        keys, values = [None] * l, [None] * l
+        i = 0
+        for key, val in d.iteritems():
+            keys[i] = key
+            values[i] = val
+            i += 1
+        return keys, values
+
+    def get_storage_fromkeys(self, keys_w, w_fill):
+        """Return an initialized storage with keys and fill values"""
+        storage = {}
+        mark_dict_non_null(storage)
+        for key in keys_w:
+            storage[key] = w_fill
+        return self.erase(storage)
 
 create_iterator_classes(UnicodeDictStrategy)
 
@@ -1426,7 +1434,7 @@
         typename = space.type(self).getname(space)
         w_seq = space.call_function(space.w_list, self)
         seq_repr = space.utf8_w(space.repr(w_seq))
-        return space.newtext(b"%s(%s)" % (typename, seq_repr))
+        return space.newtext(u"%s(%s)" % (typename, seq_repr.decode('utf8')))
 
     def descr_len(self, space):
         return space.len(self.w_dict)
diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py
--- a/pypy/objspace/std/objspace.py
+++ b/pypy/objspace/std/objspace.py
@@ -327,14 +327,13 @@
         return W_ListObject.newlist_bytes(self, list_s)
 
     def newlist_text(self, list_t):
-        return self.newlist_unicode([
+        return self.newlist_utf8([
             str_decode_utf8(s, "string", True, None, allow_surrogates=True)[0]
                      for s in list_t])
 
-    def newlist_utf8(self, list_u, is_ascii):
-        if is_ascii:
-            return W_ListObject.newlist_utf8(self, list_u)
-        return ObjSpace.newlist_utf8(self, list_u, False)
+    def newlist_utf8(self, list_u, is_ascii=True):
+        # TODO ignoring is_ascii, is that correct?
+        return W_ListObject.newlist_utf8(self, list_u)
 
     def newlist_int(self, list_i):
         return W_ListObject.newlist_int(self, list_i)
@@ -553,8 +552,7 @@
             return w_obj.listview_utf8()
         if type(w_obj) is W_SetObject or type(w_obj) is W_FrozensetObject:
             return w_obj.listview_utf8()
-        if (isinstance(w_obj, W_UnicodeObject) and not self._uses_unicode_iter(w_obj)
-            and w_obj.is_ascii()):
+        if isinstance(w_obj, W_UnicodeObject) and self._uses_unicode_iter(w_obj):
             return w_obj.listview_utf8()
         if isinstance(w_obj, W_ListObject) and self._uses_list_iter(w_obj):
             return w_obj.getitems_utf8()
diff --git a/pypy/objspace/std/test/test_dictmultiobject.py b/pypy/objspace/std/test/test_dictmultiobject.py
--- a/pypy/objspace/std/test/test_dictmultiobject.py
+++ b/pypy/objspace/std/test/test_dictmultiobject.py
@@ -1247,6 +1247,11 @@
         self.hash_count += 1
         return unicode.__hash__(self)
 
+    def is_ascii(self):
+        return True
+
+    def unwrapped(self):
+        return True
 
 # the minimal 'space' needed to use a W_DictMultiObject
 class FakeSpace:
@@ -1285,15 +1290,17 @@
 
     def text_w(self, u):
         assert isinstance(u, unicode)
-        return u.encode('utf-8')
+        return FakeUnicode(u)
 
     def bytes_w(self, string):
         assert isinstance(string, str)
         return string
 
-    def utf8_w(self, b):
+    def utf8_w(self, u):
+        if isinstance(u, unicode):
+            u = u.encode('utf8')
         assert isinstance(u, str)
-        return b
+        return u
 
     def int_w(self, integer, allow_conversion=True):
         assert isinstance(integer, int)
@@ -1301,12 +1308,17 @@
 
     def wrap(self, obj):
         if isinstance(obj, str):
-            return obj.decode('ascii')
+            return FakeUnicode(obj.decode('ascii'))
         return obj
 
     def newtext(self, string):
-        assert isinstance(string, str)
-        return string.decode('utf-8')
+        if isinstance(string, str):
+            return FakeUnicode(string.decode('utf-8'))
+        assert isinstance(string, unicode)
+        return FakeUnicode(string)
+
+    def newutf8(self, obj, lgt):
+        return obj
 
     def newbytes(self, obj):
         return obj
diff --git a/pypy/objspace/std/test/test_unicodeobject.py b/pypy/objspace/std/test/test_unicodeobject.py
--- a/pypy/objspace/std/test/test_unicodeobject.py
+++ b/pypy/objspace/std/test/test_unicodeobject.py
@@ -77,22 +77,30 @@
         assert space.int_w(w_index) == rexpected
 
         expected = u.startswith(v, start)
+        if expected and start > len(u):
+            expected = False # python2 vs. python3
         w_res = space.call_method(w_u, 'startswith', w_v,
                                   space.newint(start))
         assert w_res is space.newbool(expected)
 
         expected = u.startswith(v, start, start + len1)
+        if expected and start > len(u):
+            expected = False # python2 vs. python3
         w_res = space.call_method(w_u, 'startswith', w_v,
                                   space.newint(start),
                                   space.newint(start + len1))
         assert w_res is space.newbool(expected)
 
         expected = u.endswith(v, start)
+        if expected and start > len(u):
+            expected = False # python2 vs. python3
         w_res = space.call_method(w_u, 'endswith', w_v,
                                   space.newint(start))
         assert w_res is space.newbool(expected)
 
         expected = u.endswith(v, start, start + len1)
+        if expected and start > len(u):
+            expected = False # python2 vs. python3
         w_res = space.call_method(w_u, 'endswith', w_v,
                                   space.newint(start),
                                   space.newint(start + len1))
@@ -102,6 +110,7 @@
         space = self.space
         w_uni = space.wrap(u'abcd')
         assert space.text_w(w_uni) == 'abcd'
+        # TODO : how to handle this?
         w_uni = space.wrap(unichr(0xd921) + unichr(0xdddd))
         space.raises_w(space.w_UnicodeEncodeError, space.text_w, w_uni)
 
diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -35,6 +35,7 @@
     @enforceargs(utf8str=str)
     def __init__(self, utf8str, length):
         assert isinstance(utf8str, bytes)
+        # TODO: how to handle surrogates
         assert length >= 0
         self._utf8 = utf8str
         self._length = length
@@ -125,7 +126,8 @@
         if isinstance(w_other, W_UnicodeObject):
             return w_other
         if space.isinstance_w(w_other, space.w_bytes):
-            return unicode_from_bytes(space, w_other)
+            raise oefmt(space.w_TypeError,
+                    "Can't convert '%T' object to str implicitly", w_other)
         if strict:
             raise oefmt(space.w_TypeError,
                 "%s arg must be None, unicode or str", strict)
@@ -142,8 +144,6 @@
     def _multi_chr(self, unichar):
         return unichar
 
-    _builder = UnicodeBuilder
-
     def _generic_name(self):
         return "str"
 
@@ -373,14 +373,15 @@
         return mod_format(space, w_values, self, fmt_type=FORMAT_UNICODE)
 
     def descr_swapcase(self, space):
-        input = self._utf8
-        builder = rutf8.Utf8StringBuilder(len(input))
-        for ch in rutf8.Utf8StringIterator(input):
+        value = self._utf8
+        builder = rutf8.Utf8StringBuilder(len(value))
+        for ch in rutf8.Utf8StringIterator(value):
             if unicodedb.isupper(ch):
-                ch = unicodedb.tolower(ch)
+                codes = unicodedb.tolower_full(ch)
             elif unicodedb.islower(ch):
-                ch = unicodedb.toupper(ch)
-            builder.append_code(ch)
+                codes = unicodedb.toupper_full(ch)
+            for c in codes:
+                builder.append_code(c)
         return self.from_utf8builder(builder)
 
     def descr_title(self, space):
@@ -393,15 +394,51 @@
         input = self._utf8
         builder = rutf8.Utf8StringBuilder(len(input))
         previous_is_cased = False
+        i = 0
         for ch in rutf8.Utf8StringIterator(input):
-            if not previous_is_cased:
-                ch = unicodedb.totitle(ch)
+            if ch == 0x3a3:
+                codes = [self._handle_capital_sigma(input, i),]
+            elif not previous_is_cased:
+                codes = unicodedb.totitle_full(ch)
             else:
-                ch = unicodedb.tolower(ch)
-            builder.append_code(ch)
-            previous_is_cased = unicodedb.iscased(ch)
+                codes = unicodedb.tolower_full(ch)
+            for c in codes:
+                builder.append_code(c)
+            previous_is_cased = unicodedb.iscased(codes[-1])
+            i += 1
         return self.from_utf8builder(builder)
 
+    def _handle_capital_sigma(self, value, i):
+        # U+03A3 is in the Final_Sigma context when, it is found like this:
+        #\p{cased} \p{case-ignorable}* U+03A3 not(\p{case-ignorable}* \p{cased})
+        # where \p{xxx} is a character with property xxx.
+
+        # TODO: find a better way for utf8 -> codepoints
+        value = [ch for ch in rutf8.Utf8StringIterator(value)]
+        j = i - 1
+        final_sigma = False
+        while j >= 0:
+            ch = value[j]
+            if unicodedb.iscaseignorable(ch):
+                j -= 1
+                continue
+            final_sigma = unicodedb.iscased(ch)
+            break
+        if final_sigma:
+            j = i + 1
+            length = len(value)
+            while j < length:
+                ch = value[j]
+                if unicodedb.iscaseignorable(ch):
+                    j += 1
+                    continue
+                final_sigma = not unicodedb.iscased(ch)
+                break
+        if final_sigma:
+            return 0x3C2
+        else:
+            return 0x3C3
+
     def descr_translate(self, space, w_table):
         builder = rutf8.Utf8StringBuilder(len(self._utf8))
         for codepoint in rutf8.Utf8StringIterator(self._utf8):
@@ -519,23 +556,29 @@
         return space.is_w(space.type(w_obj), space.w_unicode)
 
     def descr_casefold(self, space):
-        value = self._val(space)
-        builder = self._builder(len(value))
-        for c in value:
-            c_ord = ord(c)
-            folded = unicodedb.casefold_lookup(c_ord)
+        value = self._utf8
+        builder = rutf8.Utf8StringBuilder(len(value))
+        for ch in rutf8.Utf8StringIterator(value):
+            folded = unicodedb.casefold_lookup(ch)
             if folded is None:
-                builder.append(unichr(unicodedb.tolower(c_ord)))
+                builder.append_code(unicodedb.tolower(ch))
             else:
                 for r in folded:
-                    builder.append(unichr(r))
-        return self._new(builder.build())
+                    builder.append_code(r)
+        return self.from_utf8builder(builder)
 
     def descr_lower(self, space):
-        builder = rutf8.Utf8StringBuilder(len(self._utf8))
-        for ch in rutf8.Utf8StringIterator(self._utf8):
-            lower = unicodedb.tolower(ch)
-            builder.append_code(lower)
+        value = self._utf8
+        builder = rutf8.Utf8StringBuilder(len(value))
+        i = 0
+        for ch in rutf8.Utf8StringIterator(value):
+            if ch == 0x3a3:
+                codes = [self._handle_capital_sigma(value, i),]
+            else:
+                codes = unicodedb.tolower_full(ch)
+            for c in codes:
+                builder.append_code(c)
+            i += 1
         return self.from_utf8builder(builder)
 
     def descr_isdecimal(self, space):
@@ -589,11 +632,18 @@
         value = self._utf8
         if space.isinstance_w(w_prefix, space.w_tuple):
             return self._startswith_tuple(space, value, w_prefix, start, end)
-        return space.newbool(self._startswith(space, value, w_prefix, start,
+        try:
+            return space.newbool(self._startswith(space, value, w_prefix, start,
                                               end))
+        except OperationError as e:
+            if e.match(space, space.w_TypeError):
+                raise oefmt(space.w_TypeError, 'startswith first arg must be str '
+                        'or a tuple of str, not %T', w_prefix)
 
     def _startswith(self, space, value, w_prefix, start, end):
         prefix = self.convert_arg_to_w_unicode(space, w_prefix)._utf8
+        if start > len(value):
+            return False
         if len(prefix) == 0:
             return True
         return startswith(value, prefix, start, end)
@@ -603,11 +653,18 @@
         value = self._utf8
         if space.isinstance_w(w_suffix, space.w_tuple):
             return self._endswith_tuple(space, value, w_suffix, start, end)
-        return space.newbool(self._endswith(space, value, w_suffix, start,
+        try:
+            return space.newbool(self._endswith(space, value, w_suffix, start,
                                             end))
+        except OperationError as e:
+            if e.match(space, space.w_TypeError):
+                raise oefmt(space.w_TypeError, 'endswith first arg must be str '
+                        'or a tuple of str, not %T', w_suffix)
 
     def _endswith(self, space, value, w_prefix, start, end):
         prefix = self.convert_arg_to_w_unicode(space, w_prefix)._utf8
+        if start > len(value):
+            return False
         if len(prefix) == 0:
             return True
         return endswith(value, prefix, start, end)
@@ -684,8 +741,9 @@
     def descr_upper(self, space):
         builder = rutf8.Utf8StringBuilder(len(self._utf8))
         for ch in rutf8.Utf8StringIterator(self._utf8):
-            ch = unicodedb.toupper(ch)
-            builder.append_code(ch)
+            codes = unicodedb.toupper_full(ch)
+            for c in codes:
+                builder.append_code(c)
         return self.from_utf8builder(builder)
 
     @unwrap_spec(width=int)
@@ -792,14 +850,16 @@
         builder = rutf8.Utf8StringBuilder(len(self._utf8))
         it = rutf8.Utf8StringIterator(self._utf8)
         uchar = it.next()
-        ch = unicodedb.toupper(uchar)
-        builder.append_code(ch)
+        codes = unicodedb.toupper_full(uchar)
+        # can sometimes give more than one, like for omega-with-Ypogegrammeni, 8179
+        for c in codes:
+            builder.append_code(c)
         for ch in it:
             ch = unicodedb.tolower(ch)
             builder.append_code(ch)
         return self.from_utf8builder(builder)
 
-    @unwrap_spec(width=int, w_fillchar=WrappedDefault(' '))
+    @unwrap_spec(width=int, w_fillchar=WrappedDefault(u' '))
     def descr_center(self, space, width, w_fillchar):
         value = self._utf8
         fillchar = self.convert_arg_to_w_unicode(space, w_fillchar)._utf8
@@ -978,14 +1038,14 @@
         end_index = len(self._utf8)
         if start > 0:
             if start > self._length:
-                start_index = end_index
+                start_index = end_index + 1
             else:
                 start_index = self._index_to_byte(start)
         if end < self._length:
             end_index = self._index_to_byte(end)
         return (start_index, end_index)
 
-    @unwrap_spec(width=int, w_fillchar=WrappedDefault(' '))
+    @unwrap_spec(width=int, w_fillchar=WrappedDefault(u' '))
     def descr_rjust(self, space, width, w_fillchar):
         value = self._utf8
         lgt = self._len()
@@ -1004,7 +1064,7 @@
 
         return W_UnicodeObject(value, lgt)
 
-    @unwrap_spec(width=int, w_fillchar=WrappedDefault(' '))
+    @unwrap_spec(width=int, w_fillchar=WrappedDefault(u' '))
     def descr_ljust(self, space, width, w_fillchar):
         value = self._utf8
         w_fillchar = self.convert_arg_to_w_unicode(space, w_fillchar)
@@ -1080,23 +1140,11 @@
 
 
     def descr_isprintable(self, space):
-        for uchar in self._value:
-            if not unicodedb.isprintable(ord(uchar)):
+        for ch in rutf8.Utf8StringIterator(self._utf8):
+            if not unicodedb.isprintable(ch):
                 return space.w_False
         return space.w_True
 
-    def _fix_fillchar(func):
-        # XXX: hack
-        from rpython.tool.sourcetools import func_with_new_name
-        func = func_with_new_name(func, func.__name__)
-        func.unwrap_spec = func.unwrap_spec.copy()
-        func.unwrap_spec['w_fillchar'] = WrappedDefault(u' ')
-        return func
-
-    descr_center = _fix_fillchar(StringMethods.descr_center)
-    descr_ljust = _fix_fillchar(StringMethods.descr_ljust)
-    descr_rjust = _fix_fillchar(StringMethods.descr_rjust)
-
     @staticmethod
     def _iter_getitem_result(self, space, index):
         assert isinstance(self, W_UnicodeObject)
@@ -1172,7 +1220,7 @@
 def decode_object(space, w_obj, encoding, errors):
     if encoding is None:
         encoding = getdefaultencoding(space)
-    if errors is None or errors == 'strict':
+    if errors is None or errors == 'strict' or errors == 'surrogateescape':
         if encoding == 'ascii':
             s = space.charbuf_w(w_obj)
             unicodehelper.check_ascii_or_raise(space, s)
@@ -1824,7 +1872,7 @@
 def unicode_to_decimal_w(space, w_unistr, allow_surrogates=False):
     if not isinstance(w_unistr, W_UnicodeObject):
         raise oefmt(space.w_TypeError, "expected unicode, got '%T'", w_unistr)
-    value = _rpy_unicode_to_decimal_w(space, w_unistr.utf8_w(space))
+    value = _rpy_unicode_to_decimal_w(space, w_unistr.utf8_w(space).decode('utf8'))
     return unicodehelper.encode_utf8(space, value,
                                      allow_surrogates=allow_surrogates)
 

From pypy.commits at gmail.com  Mon Jul  9 09:01:00 2018
From: pypy.commits at gmail.com (mattip)
Date: Mon, 09 Jul 2018 06:01:00 -0700 (PDT)
Subject: [pypy-commit] pypy unicode-utf8-py3: utf_8 -> utf8_w,
 str_w -> text_w
Message-ID: <5b435c8c.1c69fb81.f1a1c.2b9d@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: unicode-utf8-py3
Changeset: r94836:88b5b1b85e78
Date: 2018-07-09 05:10 -0700
http://bitbucket.org/pypy/pypy/changeset/88b5b1b85e78/

Log:	utf_8 -> utf8_w, str_w -> text_w

diff --git a/pypy/interpreter/generator.py b/pypy/interpreter/generator.py
--- a/pypy/interpreter/generator.py
+++ b/pypy/interpreter/generator.py
@@ -312,7 +312,7 @@
 
     def descr_set__qualname__(self, space, w_name):
         try:
-            self._qualname = space.utf_8(w_name)
+            self._qualname = space.utf8_w(w_name)
         except OperationError as e:
             if e.match(space, space.w_TypeError):
                 raise oefmt(space.w_TypeError,
diff --git a/pypy/interpreter/pycode.py b/pypy/interpreter/pycode.py
--- a/pypy/interpreter/pycode.py
+++ b/pypy/interpreter/pycode.py
@@ -206,7 +206,7 @@
         self.co_filename = '<builtin>/%s' % (basename,)
         self.w_filename = self.space.newfilename(self.co_filename)
 
-    co_names = property(lambda self: [self.space.str_w(w_name) for w_name in self.co_names_w]) # for trace
+    co_names = property(lambda self: [self.space.text_w(w_name) for w_name in self.co_names_w]) # for trace
 
     def signature(self):
         return self._signature
@@ -453,7 +453,7 @@
         space = self.space
         # co_name should be an identifier
         name = self.co_name.decode('utf-8')
-        fn = space.utf_8(self.w_filename)
+        fn = space.utf8_w(self.w_filename)
         return space.newtext(b'<code object %s at 0x%s, file "%s", line %d>' % (
             name, unicode(self.getaddrstring(space)), fn,
             -1 if self.co_firstlineno == 0 else self.co_firstlineno))
diff --git a/pypy/interpreter/test/test_appinterp.py b/pypy/interpreter/test/test_appinterp.py
--- a/pypy/interpreter/test/test_appinterp.py
+++ b/pypy/interpreter/test/test_appinterp.py
@@ -155,7 +155,7 @@
         w_mymod2 = MyModule(space2, space2.wrap('mymod'))
 
         w_str = space1.getattr(w_mymod1, space1.wrap("hi"))
-        assert space1.str_w(w_str) == "hello"
+        assert space1.text_w(w_str) == "hello"
 
 class TestMixedModuleUnfreeze:
     spaceconfig = dict(usemodules=('_socket',))
diff --git a/pypy/interpreter/test/test_argument.py b/pypy/interpreter/test/test_argument.py
--- a/pypy/interpreter/test/test_argument.py
+++ b/pypy/interpreter/test/test_argument.py
@@ -99,6 +99,9 @@
     def text_w(self, s):
         return self.utf8_w(s)
 
+    def utf8_w(self, s):
+        return s
+
     def len(self, x):
         return len(x)
 
@@ -668,14 +671,14 @@
         try:
             Arguments(space, [], w_stararg=space.wrap(42))
         except OperationError as e:
-            msg = space.str_w(space.str(e.get_w_value(space)))
+            msg = space.text_w(space.str(e.get_w_value(space)))
             assert msg == "argument after * must be an iterable, not int"
         else:
             assert 0, "did not raise"
         try:
             Arguments(space, [], w_starstararg=space.wrap(42))
         except OperationError as e:
-            msg = space.str_w(space.str(e.get_w_value(space)))
+            msg = space.text_w(space.str(e.get_w_value(space)))
             assert msg == "argument after ** must be a mapping, not int"
         else:
             assert 0, "did not raise"
diff --git a/pypy/interpreter/test/test_compiler.py b/pypy/interpreter/test/test_compiler.py
--- a/pypy/interpreter/test/test_compiler.py
+++ b/pypy/interpreter/test/test_compiler.py
@@ -664,7 +664,7 @@
         w_d = space.newdict()
         space.exec_(code, w_d, w_d)
         w_res = space.getitem(w_d, space.wrap('res'))
-        assert space.str_w(w_res) == "global value"
+        assert space.text_w(w_res) == "global value"
 
     def test_method_and_var(self):
         space = self.space
@@ -753,7 +753,7 @@
         ex = e.value
         space = self.space
         assert ex.match(space, space.w_SyntaxError)
-        assert 'hello_world' in space.str_w(space.str(ex.get_w_value(space)))
+        assert 'hello_world' in space.text_w(space.str(ex.get_w_value(space)))
 
     def test_del_None(self):
         snippet = '''if 1:
diff --git a/pypy/interpreter/test/test_error.py b/pypy/interpreter/test/test_error.py
--- a/pypy/interpreter/test/test_error.py
+++ b/pypy/interpreter/test/test_error.py
@@ -43,7 +43,7 @@
     val = operr.get_w_value(space)
     assert space.isinstance_w(val, space.w_AttributeError)
     w_repr = space.repr(val)
-    assert space.str_w(w_repr) == "AttributeError(\"no attribute 'foo'\",)"
+    assert space.text_w(w_repr) == "AttributeError(\"no attribute 'foo'\",)"
 
 def test_oefmt_T(space):
     operr = oefmt(space.w_AttributeError,
@@ -167,7 +167,7 @@
 def test_new_exception(space):
     w_error = new_exception_class(space, '_socket.error')
     assert w_error.getname(space) == u'error'
-    assert space.str_w(space.repr(w_error)) == "<class '_socket.error'>"
+    assert space.text_w(space.repr(w_error)) == "<class '_socket.error'>"
     operr = OperationError(w_error, space.wrap("message"))
     assert operr.match(space, w_error)
     assert operr.match(space, space.w_Exception)
diff --git a/pypy/interpreter/test/test_gateway.py b/pypy/interpreter/test/test_gateway.py
--- a/pypy/interpreter/test/test_gateway.py
+++ b/pypy/interpreter/test/test_gateway.py
@@ -204,7 +204,7 @@
         assert space.int_w(space.call_function(w_c, w_a, space.wrap(1))) == 1 + 2
         assert space.int_w(space.call_function(w_c, w_b, space.wrap(-10))) == -10 + 1
 
-        doc = space.str_w(space.getattr(w_c, space.wrap('__doc__')))
+        doc = space.text_w(space.getattr(w_c, space.wrap('__doc__')))
         assert doc == "This is a method"
 
         meth_with_default = gateway.interpindirect2app(
@@ -856,7 +856,7 @@
             except SystemError as e:
                 return str(e)
         """)
-        err = space.str_w(w_msg)
+        err = space.text_w(w_msg)
         assert ('unexpected internal exception (please '
                 'report a bug): UnexpectedException') in err
 
diff --git a/pypy/interpreter/test/test_objspace.py b/pypy/interpreter/test/test_objspace.py
--- a/pypy/interpreter/test/test_objspace.py
+++ b/pypy/interpreter/test/test_objspace.py
@@ -130,13 +130,13 @@
         w_object_doc = self.space.getattr(self.space.w_object, w("__doc__"))
         w_instance = self.space.appexec([], "(): return object()")
         w_doc = self.space.lookup(w_instance, "__doc__")
-        assert self.space.str_w(w_doc) == self.space.str_w(w_object_doc)
+        assert self.space.text_w(w_doc) == self.space.text_w(w_object_doc)
         assert self.space.lookup(w_instance, "gobbledygook") is None
         w_instance = self.space.appexec([], """():
             class Lookup(object):
                 "bla"
             return Lookup()""")
-        assert self.space.str_w(self.space.lookup(w_instance, "__doc__")) == "bla"
+        assert self.space.text_w(self.space.lookup(w_instance, "__doc__")) == "bla"
 
     def test_callable(self):
         def is_callable(w_obj):
@@ -404,9 +404,9 @@
         w_executable = space.wrap('executable')
         assert space.findattr(space.sys, w_executable) is None
         space.setattr(space.sys, w_executable, space.wrap('foobar'))
-        assert space.str_w(space.getattr(space.sys, w_executable)) == 'foobar'
+        assert space.text_w(space.getattr(space.sys, w_executable)) == 'foobar'
         space.startup()
-        assert space.str_w(space.getattr(space.sys, w_executable)) == 'foobar'
+        assert space.text_w(space.getattr(space.sys, w_executable)) == 'foobar'
 
     def test_interned_strings_are_weak(self):
         import weakref, gc, random
diff --git a/pypy/interpreter/test/test_typedef.py b/pypy/interpreter/test/test_typedef.py
--- a/pypy/interpreter/test/test_typedef.py
+++ b/pypy/interpreter/test/test_typedef.py
@@ -200,7 +200,7 @@
         w_seen = space.newlist([])
         W_Level1(space)
         gc.collect(); gc.collect()
-        assert space.str_w(space.repr(w_seen)) == "[]"  # not called yet
+        assert space.text_w(space.repr(w_seen)) == "[]"  # not called yet
         ec = space.getexecutioncontext()
         self.space.user_del_action.perform(ec, None)
         assert space.unwrap(w_seen) == [1]   # called by user_del_action
diff --git a/pypy/module/_cffi_backend/test/test_c.py b/pypy/module/_cffi_backend/test/test_c.py
--- a/pypy/module/_cffi_backend/test/test_c.py
+++ b/pypy/module/_cffi_backend/test/test_c.py
@@ -76,7 +76,7 @@
                     path = None
                 else:
                     import ctypes.util
-                    path = ctypes.util.find_library(space.str_w(w_name))
+                    path = ctypes.util.find_library(space.text_w(w_name))
                 return space.appexec([space.wrap(path), w_is_global],
                 """(path, is_global):
                     import _cffi_backend
diff --git a/pypy/module/_cffi_backend/test/test_recompiler.py b/pypy/module/_cffi_backend/test/test_recompiler.py
--- a/pypy/module/_cffi_backend/test/test_recompiler.py
+++ b/pypy/module/_cffi_backend/test/test_recompiler.py
@@ -55,7 +55,7 @@
     base_module_name = module_name.split('.')[-1]
     sources = []
     if w_extra_source is not None:
-        sources.append(space.str_w(w_extra_source))
+        sources.append(space.text_w(w_extra_source))
     kwargs = {}
     if w_extra_compile_args is not None:
         kwargs['extra_compile_args'] = space.unwrap(w_extra_compile_args)
diff --git a/pypy/module/pyexpat/interp_pyexpat.py b/pypy/module/pyexpat/interp_pyexpat.py
--- a/pypy/module/pyexpat/interp_pyexpat.py
+++ b/pypy/module/pyexpat/interp_pyexpat.py
@@ -2,7 +2,6 @@
 from pypy.interpreter.typedef import TypeDef, GetSetProperty
 from pypy.interpreter.gateway import interp2app, unwrap_spec, WrappedDefault
 from pypy.interpreter.error import OperationError, oefmt
-from pypy.interpreter.unicodehelper import encode_utf8
 from rpython.rlib import rgc, jit, rutf8
 from rpython.rlib.objectmodel import specialize
 from rpython.rtyper.lltypesystem import rffi, lltype
@@ -639,7 +638,7 @@
         """Parse(data[, isfinal])
 Parse XML data.  `isfinal' should be true at end of input."""
         if space.isinstance_w(w_data, space.w_unicode):
-            data = encode_utf8(space, w_data.utf8_w(space))
+            data = w_data.utf8_w(space)
             # Explicitly set UTF-8 encoding. Return code ignored.
             XML_SetEncoding(self.itself, "utf-8")
         else:
diff --git a/pypy/tool/pydis.py b/pypy/tool/pydis.py
--- a/pypy/tool/pydis.py
+++ b/pypy/tool/pydis.py
@@ -69,7 +69,7 @@
         if space is None:
             return [repr(c) for c in co.co_consts_w]
         
-        r = lambda x: space.str_w(space.repr(x))
+        r = lambda x: space.text_w(space.repr(x))
         return [r(c) for c in co.co_consts_w]
 
     def repr_with_space(self, space):

From pypy.commits at gmail.com  Mon Jul  9 09:01:02 2018
From: pypy.commits at gmail.com (mattip)
Date: Mon, 09 Jul 2018 06:01:02 -0700 (PDT)
Subject: [pypy-commit] pypy unicode-utf8-py3: start to pass unicode around,
 try using decode_utf8sp
Message-ID: <5b435c8e.1c69fb81.7d4a3.f1f3@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: unicode-utf8-py3
Changeset: r94837:2621c0f70d91
Date: 2018-07-09 05:11 -0700
http://bitbucket.org/pypy/pypy/changeset/2621c0f70d91/

Log:	start to pass unicode around, try using decode_utf8sp

diff --git a/pypy/interpreter/argument.py b/pypy/interpreter/argument.py
--- a/pypy/interpreter/argument.py
+++ b/pypy/interpreter/argument.py
@@ -602,8 +602,8 @@
 
     def getmsg(self):
         if self.num_kwds == 1:
-            msg = "got an unexpected keyword argument '%s'" % (
-                self.kwd_name)
+            msg = u"got an unexpected keyword argument '%s'" % (
+                self.kwd_name.decode('utf8'))
         else:
             msg = "got %d unexpected keyword arguments" % (
                 self.num_kwds)
diff --git a/pypy/interpreter/error.py b/pypy/interpreter/error.py
--- a/pypy/interpreter/error.py
+++ b/pypy/interpreter/error.py
@@ -472,9 +472,12 @@
     assert len(formats) > 0, "unsupported: no % command found"
     return tuple(parts), tuple(formats)
 
+ at specialize.arg(1)
 def _decode_utf8(string):
     # when building the error message, don't crash if the byte string
     # provided is not valid UTF-8
+    if isinstance(string, unicode):
+        return string
     assert isinstance(string, str)
     result, consumed = runicode.str_decode_utf_8(
         string, len(string), "replace", final=True)
diff --git a/pypy/interpreter/function.py b/pypy/interpreter/function.py
--- a/pypy/interpreter/function.py
+++ b/pypy/interpreter/function.py
@@ -255,7 +255,7 @@
         return self.call_args(__args__)
 
     def descr_function_repr(self):
-        return self.getrepr(self.space, u'function %s' % self.qualname)
+        return self.getrepr(self.space, u'function %s' % self.qualname.decode('utf8'))
 
 
     def _cleanup_(self):
diff --git a/pypy/interpreter/unicodehelper.py b/pypy/interpreter/unicodehelper.py
--- a/pypy/interpreter/unicodehelper.py
+++ b/pypy/interpreter/unicodehelper.py
@@ -155,6 +155,7 @@
     try:
         rutf8.check_ascii(string)
     except rutf8.CheckError as e:
+        print 'check_ascii_or_raise', string
         decode_error_handler(space)('strict', 'ascii',
                                     'ordinal not in range(128)', string,
                                     e.pos, e.pos + 1)
diff --git a/pypy/module/_codecs/interp_codecs.py b/pypy/module/_codecs/interp_codecs.py
--- a/pypy/module/_codecs/interp_codecs.py
+++ b/pypy/module/_codecs/interp_codecs.py
@@ -638,7 +638,7 @@
         state = space.fromcache(CodecState)
         utf8len = w_arg._length
         # XXX deal with func() returning length or not
-        result = func(w_arg._utf8, errors, state.encode_error_handler)
+        result = func(w_arg._utf8.decode('utf8'), errors, state.encode_error_handler)
         return space.newtuple([space.newbytes(result), space.newint(utf8len)])
     wrap_encoder.__name__ = func.__name__
     globals()[name] = wrap_encoder
diff --git a/pypy/module/time/interp_time.py b/pypy/module/time/interp_time.py
--- a/pypy/module/time/interp_time.py
+++ b/pypy/module/time/interp_time.py
@@ -5,7 +5,7 @@
 from pypy.interpreter.gateway import unwrap_spec
 from pypy.interpreter.timeutils import (
     SECS_TO_NS, MS_TO_NS, US_TO_NS, monotonic as _monotonic, timestamp_w)
-from pypy.interpreter.unicodehelper import str_decode_utf8
+from pypy.interpreter.unicodehelper import decode_utf8sp
 from rpython.rtyper.lltypesystem import lltype
 from rpython.rlib.rarithmetic import (
     intmask, r_ulonglong, r_longfloat, widen, ovfcheck, ovfcheck_float_to_int)
@@ -554,8 +554,7 @@
 
     if HAS_TM_ZONE:
         # CPython calls PyUnicode_DecodeLocale here should we do the same?
-        tm_zone = str_decode_utf8(rffi.charp2str(t.c_tm_zone),
-                              allow_surrogates=True)
+        tm_zone = decode_utf8sp(space, rffi.charp2str(t.c_tm_zone))
         extra = [space.newtext(tm_zone),
                  space.newint(rffi.getintfield(t, 'c_tm_gmtoff'))]
         w_time_tuple = space.newtuple(time_tuple + extra)
diff --git a/pypy/objspace/std/dictmultiobject.py b/pypy/objspace/std/dictmultiobject.py
--- a/pypy/objspace/std/dictmultiobject.py
+++ b/pypy/objspace/std/dictmultiobject.py
@@ -12,7 +12,7 @@
 from pypy.interpreter.mixedmodule import MixedModule
 from pypy.interpreter.signature import Signature
 from pypy.interpreter.typedef import TypeDef
-from pypy.interpreter.unicodehelper import str_decode_utf8
+from pypy.interpreter.unicodehelper import decode_utf8sp
 from pypy.objspace.std.util import negate
 
 
@@ -1183,9 +1183,12 @@
 
     # we should implement the same shortcuts as we do for BytesDictStrategy
 
+    def decodekey_str(self, key):
+        return decode_utf8sp(self.space, key)[0]
+
     def setitem_str(self, w_dict, key, w_value):
         assert key is not None
-        self.unerase(w_dict.dstorage)[key] = w_value
+        self.unerase(w_dict.dstorage)[self.decodekey_str(key)] = w_value
 
     def getitem(self, w_dict, w_key):
         space = self.space
@@ -1197,7 +1200,7 @@
 
     def getitem_str(self, w_dict, key):
         assert key is not None
-        return self.unerase(w_dict.dstorage).get(key, None)
+        return self.unerase(w_dict.dstorage).get(self.decodekey_str(key), None)
 
     def listview_utf8(self, w_dict):
         return self.unerase(w_dict.dstorage).keys()
diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py
--- a/pypy/objspace/std/objspace.py
+++ b/pypy/objspace/std/objspace.py
@@ -4,7 +4,7 @@
 from pypy.interpreter.error import OperationError, oefmt
 from pypy.interpreter.function import Function, Method, FunctionWithFixedCode
 from pypy.interpreter.typedef import get_unique_interplevel_subclass
-from pypy.interpreter.unicodehelper import str_decode_utf8
+from pypy.interpreter.unicodehelper import decode_utf8sp
 from pypy.objspace.std import frame, transparent, callmethod
 from pypy.objspace.descroperation import (
     DescrOperation, get_attribute_name, raiseattrerror)
@@ -327,9 +327,7 @@
         return W_ListObject.newlist_bytes(self, list_s)
 
     def newlist_text(self, list_t):
-        return self.newlist_utf8([
-            str_decode_utf8(s, "string", True, None, allow_surrogates=True)[0]
-                     for s in list_t])
+        return self.newlist_utf8([decode_utf8sp(self, s) for s in list_t])
 
     def newlist_utf8(self, list_u, is_ascii=True):
         # TODO ignoring is_ascii, is that correct?
@@ -386,8 +384,7 @@
         if isinstance(s, unicode):
             s, lgt = s.encode('utf8'), len(s)
         elif isinstance(s, str):
-            s, uf8lgt, lgt = str_decode_utf8(s, "string", True, None,
-                                           allow_surrogates=True)
+            s, uf8lgt, lgt = decode_utf8sp(self, s)
         elif isinstance(s, tuple):
             # result of decode_utf8
             s, utf8lgt, lgt = s

From pypy.commits at gmail.com  Mon Jul  9 09:01:04 2018
From: pypy.commits at gmail.com (mattip)
Date: Mon, 09 Jul 2018 06:01:04 -0700 (PDT)
Subject: [pypy-commit] pypy unicode-utf8-py3: revert interp_sre.py to py3.5
Message-ID: <5b435c90.1c69fb81.5523d.a477@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: unicode-utf8-py3
Changeset: r94838:4ef833b2310d
Date: 2018-07-09 05:35 -0700
http://bitbucket.org/pypy/pypy/changeset/4ef833b2310d/

Log:	revert interp_sre.py to py3.5

diff --git a/pypy/module/_sre/interp_sre.py b/pypy/module/_sre/interp_sre.py
--- a/pypy/module/_sre/interp_sre.py
+++ b/pypy/module/_sre/interp_sre.py
@@ -6,14 +6,14 @@
 from pypy.interpreter.gateway import interp2app, unwrap_spec, WrappedDefault
 from pypy.interpreter.error import OperationError, oefmt
 from rpython.rlib.rarithmetic import intmask
-from rpython.rlib import jit, rutf8
-from rpython.rlib.rstring import StringBuilder
+from rpython.rlib import jit
+from rpython.rlib.rstring import StringBuilder, UnicodeBuilder
 
 # ____________________________________________________________
 #
 # Constants and exposed functions
 
-from rpython.rlib.rsre import rsre_core, rsre_char, rsre_utf8
+from rpython.rlib.rsre import rsre_core, rsre_char
 from rpython.rlib.rsre.rsre_char import CODESIZE, MAXREPEAT, MAXGROUPS, getlower, set_unicode_db
 
 
@@ -34,21 +34,14 @@
 
 
 def slice_w(space, ctx, start, end, w_default):
-    # 'start' and 'end' are byte positions
-    if ctx.ZERO <= start <= end:
+    if 0 <= start <= end:
         if isinstance(ctx, rsre_core.BufMatchContext):
             return space.newbytes(ctx._buffer.getslice(start, end, 1,
                                                         end-start))
         if isinstance(ctx, rsre_core.StrMatchContext):
-            start = ctx._real_pos(start)
-            end = ctx._real_pos(end)
             return space.newbytes(ctx._string[start:end])
-        elif isinstance(ctx, rsre_utf8.Utf8MatchContext):
-            s = ctx._utf8[start:end]
-            lgt = rutf8.get_utf8_length(s)
-            return space.newutf8(s, lgt)
         elif isinstance(ctx, rsre_core.UnicodeMatchContext):
-            return space.newtext(ctx._unicodestr[start:end])
+            return space.newunicode(ctx._unicodestr[start:end])
         else:
             # unreachable
             raise SystemError
@@ -60,7 +53,6 @@
     # Returns a list of RPython-level integers.
     # Unlike the app-level groups() method, groups are numbered from 0
     # and the returned list does not start with the whole match range.
-    # The integers are byte positions, not character indexes (for utf8).
     if num_groups == 0:
         return None
     result = [-1] * (2 * num_groups)
@@ -113,7 +105,7 @@
 
     def repr_w(self):
         space = self.space
-        u = space.utf8_w(space.repr(self.w_pattern)).decode()
+        u = space.realunicode_w(space.repr(self.w_pattern))
         if len(u) > 200:
             u = u[:200]
         flag_items = []
@@ -136,7 +128,7 @@
         else:
             usep = u', '
             uflags = u'|'.join([item.decode('latin-1') for item in flag_items])
-        return space.newtext(u're.compile(%s%s%s)' % (u, usep, uflags))
+        return space.newunicode(u're.compile(%s%s%s)' % (u, usep, uflags))
 
     def fget_groupindex(self, space):
         w_groupindex = self.w_groupindex
@@ -166,7 +158,7 @@
         buf = None
         space = self.space
         if space.isinstance_w(w_string, space.w_unicode):
-            unicodestr = space.utf8_w(w_string).decode('utf8')
+            unicodestr = space.realunicode_w(w_string)
             length = len(unicodestr)
         elif space.isinstance_w(w_string, space.w_bytes):
             string = space.bytes_w(w_string)
@@ -178,7 +170,7 @@
         return (length, unicodestr, string, buf)
 
     def make_ctx(self, w_string, pos=0, endpos=sys.maxint, flags=0):
-        """Make a StrMatchContext, BufMatchContext or a Utf8MatchContext for
+        """Make a StrMatchContext, BufMatchContext or a UnicodeMatchContext for
         searching in the given w_string object."""
         space = self.space
         length, unicodestr, string, buf = self.getstring(w_string)
@@ -211,27 +203,6 @@
                 return rsre_core.BufMatchContext(buf,
                                                  pos, endpos, flags)
 
-    def fresh_copy(self, ctx):
-        if isinstance(ctx, rsre_utf8.Utf8MatchContext):
-            result = rsre_utf8.Utf8MatchContext(
-                ctx._utf8, ctx.match_start, ctx.end, ctx.flags)
-            result.w_unicode_obj = ctx.w_unicode_obj
-        elif isinstance(ctx, rsre_core.StrMatchContext):
-            result = self._make_str_match_context(
-                ctx._string, ctx.match_start, ctx.end)
-        elif isinstance(ctx, rsre_core.BufMatchContext):
-            result = rsre_core.BufMatchContext(
-                ctx._buffer, ctx.match_start, ctx.end, ctx.flags)
-        else:
-            raise AssertionError("bad ctx type")
-        result.match_end = ctx.match_end
-        return result
-
-    def _make_str_match_context(self, str, pos, endpos):
-        # for tests to override
-        return rsre_core.StrMatchContext(str,
-                                         pos, endpos, self.flags)
-
     def getmatch(self, ctx, found):
         if found:
             return W_SRE_Match(self, ctx)
@@ -259,7 +230,7 @@
         space = self.space
         matchlist_w = []
         ctx = self.make_ctx(w_string, pos, endpos)
-        while True:
+        while ctx.match_start <= ctx.end:
             if not searchcontext(space, ctx, self.code):
                 break
             num_groups = self.num_groups
@@ -276,12 +247,8 @@
                     w_item = allgroups_w(space, ctx, fmarks, num_groups,
                                          w_emptystr)
             matchlist_w.append(w_item)
-            reset_at = ctx.match_end
-            if ctx.match_start == ctx.match_end:
-                if reset_at == ctx.end:
-                    break
-                reset_at = ctx.next_indirect(reset_at)
-            ctx.reset(reset_at)
+            no_progress = (ctx.match_start == ctx.match_end)
+            ctx.reset(ctx.match_end + no_progress)
         return space.newlist(matchlist_w)
 
     @unwrap_spec(pos=int, endpos=int)
@@ -306,15 +273,15 @@
         #
         splitlist = []
         n = 0
+        last = 0
         ctx = self.make_ctx(w_string)
-        last = ctx.ZERO
         while not maxsplit or n < maxsplit:
             if not searchcontext(space, ctx, self.code):
                 break
             if ctx.match_start == ctx.match_end:     # zero-width match
                 if ctx.match_start == ctx.end:       # or end of string
                     break
-                ctx.reset(ctx.next_indirect(ctx.match_end))
+                ctx.reset(ctx.match_end + 1)
                 continue
             splitlist.append(slice_w(space, ctx, last, ctx.match_start,
                                      space.w_None))
@@ -343,31 +310,27 @@
 
     def subx(self, w_ptemplate, w_string, count):
         space = self.space
-        # use a (much faster) string builder (possibly utf8) if w_ptemplate and
+        # use a (much faster) string/unicode builder if w_ptemplate and
         # w_string are both string or both unicode objects, and if w_ptemplate
         # is a literal
-        use_builder = '\x00'   # or 'S'tring or 'U'nicode/UTF8
-        filter_as_string = None
+        use_builder = False
+        filter_as_unicode = filter_as_string = None
         if space.is_true(space.callable(w_ptemplate)):
             w_filter = w_ptemplate
             filter_is_callable = True
         else:
-            if space.isinstance_w(w_ptemplate, space.w_unicode):
-                filter_as_string = space.utf8_w(w_ptemplate)
+            length, filter_as_unicode, filter_as_string, buf = (
+                self.getstring(w_ptemplate))
+            if filter_as_unicode is not None:
+                literal = u'\\' not in filter_as_unicode
+                use_builder = (
+                    space.isinstance_w(w_string, space.w_unicode) and literal)
+            else:
+                if buf is not None:
+                    filter_as_string = buf.as_str()
                 literal = '\\' not in filter_as_string
-                if space.isinstance_w(w_string, space.w_unicode) and literal:
-                    use_builder = 'U'
-            else:
-                try:
-                    filter_as_string = space.bytes_w(w_ptemplate)
-                except OperationError as e:
-                    if e.async(space):
-                        raise
-                    literal = False
-                else:
-                    literal = '\\' not in filter_as_string
-                    if space.isinstance_w(w_string, space.w_bytes) and literal:
-                        use_builder = 'S'
+                use_builder = (
+                    space.isinstance_w(w_string, space.w_bytes) and literal)
             if literal:
                 w_filter = w_ptemplate
                 filter_is_callable = False
@@ -384,16 +347,18 @@
         #
         # XXX this is a bit of a mess, but it improves performance a lot
         ctx = self.make_ctx(w_string)
-        sublist_w = strbuilder = None
-        if use_builder != '\x00':
-            assert filter_as_string is not None
-            strbuilder = StringBuilder(ctx.end)
+        sublist_w = strbuilder = unicodebuilder = None
+        if use_builder:
+            if filter_as_unicode is not None:
+                unicodebuilder = UnicodeBuilder(ctx.end)
+            else:
+                assert filter_as_string is not None
+                strbuilder = StringBuilder(ctx.end)
         else:
             sublist_w = []
-        n = 0
-        last_pos = ctx.ZERO
+        n = last_pos = 0
+        pattern = self.code
         while not count or n < count:
-            pattern = self.code
             sub_jitdriver.jit_merge_point(
                 self=self,
                 use_builder=use_builder,
@@ -402,7 +367,9 @@
                 ctx=ctx, pattern=pattern,
                 w_filter=w_filter,
                 strbuilder=strbuilder,
+                unicodebuilder=unicodebuilder,
                 filter_as_string=filter_as_string,
+                filter_as_unicode=filter_as_unicode,
                 count=count,
                 w_string=w_string,
                 n=n, last_pos=last_pos, sublist_w=sublist_w
@@ -413,7 +380,10 @@
             if last_pos < ctx.match_start:
                 _sub_append_slice(
                     ctx, space, use_builder, sublist_w,
-                    strbuilder, last_pos, ctx.match_start)
+                    strbuilder, unicodebuilder, last_pos, ctx.match_start)
+            start = ctx.match_end
+            if start == ctx.match_start:
+                start += 1
             if not (last_pos == ctx.match_start
                              == ctx.match_end and n > 0):
                 # the above ignores empty matches on latest position
@@ -421,48 +391,40 @@
                 if filter_is_callable:
                     w_match = self.getmatch(ctx, True)
                     # make a copy of 'ctx'; see test_sub_matches_stay_valid
-                    ctx = self.fresh_copy(ctx)
+                    ctx = ctx.fresh_copy(start) # match_start/match_end dropped
                     w_piece = space.call_function(w_filter, w_match)
                     if not space.is_w(w_piece, space.w_None):
-                        assert strbuilder is None
-                        assert use_builder == '\x00'
+                        assert strbuilder is None and unicodebuilder is None
+                        assert not use_builder
                         sublist_w.append(w_piece)
                 else:
-                    if use_builder != '\x00':
-                        assert filter_as_string is not None
-                        assert strbuilder is not None
-                        strbuilder.append(filter_as_string)
+                    if use_builder:
+                        if strbuilder is not None:
+                            assert filter_as_string is not None
+                            strbuilder.append(filter_as_string)
+                        else:
+                            assert unicodebuilder is not None
+                            assert filter_as_unicode is not None
+                            unicodebuilder.append(filter_as_unicode)
                     else:
                         sublist_w.append(w_filter)
                 n += 1
             elif last_pos >= ctx.end:
                 break    # empty match at the end: finished
-
-            start = ctx.match_end
-            if start == ctx.match_start:
-                if start == ctx.end:
-                    break
-                start = ctx.next_indirect(start)
             ctx.reset(start)
 
         if last_pos < ctx.end:
             _sub_append_slice(ctx, space, use_builder, sublist_w,
-                              strbuilder, last_pos, ctx.end)
-        if use_builder != '\x00':
-            assert strbuilder is not None
-            result_bytes = strbuilder.build()
-            if use_builder == 'S':
-                assert not isinstance(ctx, rsre_utf8.Utf8MatchContext)
-                return space.newbytes(result_bytes), n
-            elif use_builder == 'U':
-                assert isinstance(ctx, rsre_utf8.Utf8MatchContext)
-                return space.newutf8(result_bytes,
-                                     rutf8.get_utf8_length(result_bytes)), n
+                              strbuilder, unicodebuilder, last_pos, ctx.end)
+        if use_builder:
+            if strbuilder is not None:
+                return space.newbytes(strbuilder.build()), n
             else:
-                raise AssertionError(use_builder)
+                assert unicodebuilder is not None
+                return space.newunicode(unicodebuilder.build()), n
         else:
             if space.isinstance_w(w_string, space.w_unicode):
-                w_emptystr = space.newutf8('', 0)
+                w_emptystr = space.newunicode(u'')
             else:
                 w_emptystr = space.newbytes('')
             w_item = space.call_method(w_emptystr, 'join',
@@ -472,28 +434,26 @@
 sub_jitdriver = jit.JitDriver(
     reds="""count n last_pos
             ctx w_filter
-            strbuilder
+            strbuilder unicodebuilder
             filter_as_string
+            filter_as_unicode
             w_string sublist_w
             self""".split(),
     greens=["filter_is_callable", "use_builder", "filter_type", "pattern"])
 
 
 def _sub_append_slice(ctx, space, use_builder, sublist_w,
-                      strbuilder, start, end):
-    if use_builder != '\x00':
-        assert strbuilder is not None
+                      strbuilder, unicodebuilder, start, end):
+    if use_builder:
         if isinstance(ctx, rsre_core.BufMatchContext):
-            assert use_builder == 'S'
+            assert strbuilder is not None
             return strbuilder.append(ctx._buffer.getslice(start, end, 1, end-start))
         if isinstance(ctx, rsre_core.StrMatchContext):
-            assert use_builder == 'S'
-            start = ctx._real_pos(start)
-            end = ctx._real_pos(end)
+            assert strbuilder is not None
             return strbuilder.append_slice(ctx._string, start, end)
-        elif isinstance(ctx, rsre_utf8.Utf8MatchContext):
-            assert use_builder == 'U'
-            return strbuilder.append_slice(ctx._utf8, start, end)
+        elif isinstance(ctx, rsre_core.UnicodeMatchContext):
+            assert unicodebuilder is not None
+            return unicodebuilder.append_slice(ctx._unicodestr, start, end)
         assert 0, "unreachable"
     else:
         sublist_w.append(slice_w(space, ctx, start, end, space.w_None))
@@ -568,10 +528,10 @@
         ctx = self.ctx
         start, end = ctx.match_start, ctx.match_end
         w_s = slice_w(space, ctx, start, end, space.w_None)
-        u = space.utf8_w(space.repr(w_s)).decode()
+        u = space.realuicode_w(space.repr(w_s))
         if len(u) > 50:
             u = u[:50]
-        return space.newtext(u'<_sre.SRE_Match object; span=(%d, %d), match=%s>' %
+        return space.newunicode(u'<_sre.SRE_Match object; span=(%d, %d), match=%s>' %
                           (start, end, u))
 
     def cannot_copy_w(self):
@@ -629,38 +589,19 @@
     @unwrap_spec(w_groupnum=WrappedDefault(0))
     def start_w(self, w_groupnum):
         start, end = self.do_span(w_groupnum)
-        start = self.bytepos_to_charindex(start)
         return self.space.newint(start)
 
     @unwrap_spec(w_groupnum=WrappedDefault(0))
     def end_w(self, w_groupnum):
         start, end = self.do_span(w_groupnum)
-        end = self.bytepos_to_charindex(end)
         return self.space.newint(end)
 
     @unwrap_spec(w_groupnum=WrappedDefault(0))
     def span_w(self, w_groupnum):
         start, end = self.do_span(w_groupnum)
-        return self.new_charindex_tuple(start, end)
-
-    def new_charindex_tuple(self, start, end):
-        start = self.bytepos_to_charindex(start)
-        end = self.bytepos_to_charindex(end)
         return self.space.newtuple([self.space.newint(start),
                                     self.space.newint(end)])
 
-    def bytepos_to_charindex(self, bytepos):
-        # Transform a 'byte position', as returned by all methods from
-        # rsre_core, back into a 'character index'.  This is for UTF8
-        # handling.
-        ctx = self.ctx
-        if isinstance(ctx, rsre_utf8.Utf8MatchContext):
-            index_storage = ctx.w_unicode_obj._get_index_storage()
-            return rutf8.codepoint_index_at_byte_position(
-                ctx.w_unicode_obj._utf8, index_storage, bytepos)
-        else:
-            return bytepos
-
     def flatten_marks(self):
         if self.flatten_cache is None:
             num_groups = self.srepat.num_groups
@@ -668,8 +609,6 @@
         return self.flatten_cache
 
     def do_span(self, w_arg):
-        # return a pair of integers, which are byte positions, not
-        # character indexes (for utf8)
         space = self.space
         try:
             groupnum = space.int_w(w_arg)
@@ -717,10 +656,10 @@
         return space.w_None
 
     def fget_pos(self, space):
-        return space.newint(self.bytepos_to_charindex(self.ctx.original_pos))
+        return space.newint(self.ctx.original_pos)
 
     def fget_endpos(self, space):
-        return space.newint(self.bytepos_to_charindex(self.ctx.end))
+        return space.newint(self.ctx.end)
 
     def fget_regs(self, space):
         space = self.space
@@ -728,11 +667,11 @@
         num_groups = self.srepat.num_groups
         result_w = [None] * (num_groups + 1)
         ctx = self.ctx
-        result_w[0] = self.new_charindex_tuple(ctx.match_start,
-                                               ctx.match_end)
+        result_w[0] = space.newtuple([space.newint(ctx.match_start),
+                                      space.newint(ctx.match_end)])
         for i in range(num_groups):
-            result_w[i + 1] = self.new_charindex_tuple(fmarks[i*2],
-                                                       fmarks[i*2+1])
+            result_w[i + 1] = space.newtuple([space.newint(fmarks[i*2]),
+                                              space.newint(fmarks[i*2+1])])
         return space.newtuple(result_w)
 
     def fget_string(self, space):
@@ -741,11 +680,8 @@
             return space.newbytes(ctx._buffer.as_str())
         elif isinstance(ctx, rsre_core.StrMatchContext):
             return space.newbytes(ctx._string)
-        elif isinstance(ctx, rsre_utf8.Utf8MatchContext):
-            lgt = rutf8.get_utf8_length(ctx._utf8)
-            return space.newutf8(ctx._utf8, lgt)
         elif isinstance(ctx, rsre_core.UnicodeMatchContext):
-            return space.newtext(ctx._unicodestr)
+            return space.newunicode(ctx._unicodestr)
         else:
             raise SystemError
 
@@ -786,53 +722,38 @@
         self.ctx = ctx
         self.code = code
         # 'self.ctx' is always a fresh context in which no searching
-        # or matching succeeded so far.  It is None when the iterator is
-        # exhausted.
+        # or matching succeeded so far.
 
     def iter_w(self):
         return self
 
     def next_w(self):
-        if self.ctx is None:
+        if self.ctx.match_start > self.ctx.end:
             raise OperationError(self.space.w_StopIteration, self.space.w_None)
         if not searchcontext(self.space, self.ctx, self.code):
             raise OperationError(self.space.w_StopIteration, self.space.w_None)
         return self.getmatch(True)
 
     def match_w(self):
-        if self.ctx is None:
+        if self.ctx.match_start > self.ctx.end:
             return self.space.w_None
         return self.getmatch(matchcontext(self.space, self.ctx, self.code))
 
     def search_w(self):
-        if self.ctx is None:
+        if self.ctx.match_start > self.ctx.end:
             return self.space.w_None
         return self.getmatch(searchcontext(self.space, self.ctx, self.code))
 
     def getmatch(self, found):
-        ctx = self.ctx
-        assert ctx is not None
         if found:
+            ctx = self.ctx
             nextstart = ctx.match_end
-            exhausted = False
-            if ctx.match_start == nextstart:
-                if nextstart == ctx.end:
-                    exhausted = True
-                else:
-                    nextstart = ctx.next_indirect(nextstart)
-            if exhausted:
-                self.ctx = None
-            else:
-                self.ctx = self.srepat.fresh_copy(ctx)
-                self.ctx.match_start = nextstart
+            nextstart += (ctx.match_start == nextstart)
+            self.ctx = ctx.fresh_copy(nextstart)
             match = W_SRE_Match(self.srepat, ctx)
             return match
         else:
-            # obscure corner case
-            if ctx.match_start == ctx.end:
-                self.ctx = None
-            else:
-                ctx.match_start = ctx.next_indirect(ctx.match_start)
+            self.ctx.match_start += 1     # obscure corner case
             return None
 
 W_SRE_Scanner.typedef = TypeDef(

From pypy.commits at gmail.com  Mon Jul  9 09:01:06 2018
From: pypy.commits at gmail.com (mattip)
Date: Mon, 09 Jul 2018 06:01:06 -0700 (PDT)
Subject: [pypy-commit] pypy unicode-utf8-py3: fixes for py3.5 _sre module
Message-ID: <5b435c92.1c69fb81.cee19.398d@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: unicode-utf8-py3
Changeset: r94839:388bbf987266
Date: 2018-07-09 06:00 -0700
http://bitbucket.org/pypy/pypy/changeset/388bbf987266/

Log:	fixes for py3.5 _sre module

diff --git a/pypy/module/_codecs/interp_codecs.py b/pypy/module/_codecs/interp_codecs.py
--- a/pypy/module/_codecs/interp_codecs.py
+++ b/pypy/module/_codecs/interp_codecs.py
@@ -639,7 +639,7 @@
         utf8len = w_arg._length
         # XXX deal with func() returning length or not
         result = func(w_arg._utf8.decode('utf8'), errors, state.encode_error_handler)
-        return space.newtuple([space.newbytes(result), space.newint(utf8len)])
+        return space.newtuple([space.newbytes(result.encode('utf8')), space.newint(utf8len)])
     wrap_encoder.__name__ = func.__name__
     globals()[name] = wrap_encoder
 
diff --git a/pypy/module/_sre/interp_sre.py b/pypy/module/_sre/interp_sre.py
--- a/pypy/module/_sre/interp_sre.py
+++ b/pypy/module/_sre/interp_sre.py
@@ -41,7 +41,7 @@
         if isinstance(ctx, rsre_core.StrMatchContext):
             return space.newbytes(ctx._string[start:end])
         elif isinstance(ctx, rsre_core.UnicodeMatchContext):
-            return space.newunicode(ctx._unicodestr[start:end])
+            return space.newtext(ctx._unicodestr[start:end])
         else:
             # unreachable
             raise SystemError
@@ -128,7 +128,7 @@
         else:
             usep = u', '
             uflags = u'|'.join([item.decode('latin-1') for item in flag_items])
-        return space.newunicode(u're.compile(%s%s%s)' % (u, usep, uflags))
+        return space.newtext(u're.compile(%s%s%s)' % (u, usep, uflags))
 
     def fget_groupindex(self, space):
         w_groupindex = self.w_groupindex
@@ -421,10 +421,10 @@
                 return space.newbytes(strbuilder.build()), n
             else:
                 assert unicodebuilder is not None
-                return space.newunicode(unicodebuilder.build()), n
+                return space.newtext(unicodebuilder.build()), n
         else:
             if space.isinstance_w(w_string, space.w_unicode):
-                w_emptystr = space.newunicode(u'')
+                w_emptystr = space.newtext(u'')
             else:
                 w_emptystr = space.newbytes('')
             w_item = space.call_method(w_emptystr, 'join',
@@ -528,10 +528,10 @@
         ctx = self.ctx
         start, end = ctx.match_start, ctx.match_end
         w_s = slice_w(space, ctx, start, end, space.w_None)
-        u = space.realuicode_w(space.repr(w_s))
+        u = space.realunicode_w(space.repr(w_s))
         if len(u) > 50:
             u = u[:50]
-        return space.newunicode(u'<_sre.SRE_Match object; span=(%d, %d), match=%s>' %
+        return space.newtext(u'<_sre.SRE_Match object; span=(%d, %d), match=%s>' %
                           (start, end, u))
 
     def cannot_copy_w(self):
@@ -681,7 +681,7 @@
         elif isinstance(ctx, rsre_core.StrMatchContext):
             return space.newbytes(ctx._string)
         elif isinstance(ctx, rsre_core.UnicodeMatchContext):
-            return space.newunicode(ctx._unicodestr)
+            return space.newtext(ctx._unicodestr)
         else:
             raise SystemError
 
diff --git a/pypy/module/_sre/test/test_app_sre.py b/pypy/module/_sre/test/test_app_sre.py
--- a/pypy/module/_sre/test/test_app_sre.py
+++ b/pypy/module/_sre/test/test_app_sre.py
@@ -4,8 +4,6 @@
 import py
 from py.test import raises, skip
 from pypy.interpreter.gateway import app2interp_temp
-from pypy.module._sre import interp_sre
-from rpython.rlib.rsre.test import support
 
 
 def init_app_test(cls, space):
@@ -22,37 +20,6 @@
             sys.path.pop(0)
         """)
 
-def _test_sre_ctx_(self, str, start, end):
-    # Use the MatchContextForTests class, which handles Position
-    # instances instead of plain integers.  This is used to detect when
-    # we're accepting or escaping a Position to app-level, which we
-    # should not: Positions are meant to be byte indexes inside a
-    # possibly UTF8 string, not character indexes.
-    if not isinstance(start, support.Position):
-        start = support.Position(start)
-    if not isinstance(end, support.Position):
-        end = support.Position(end)
-    return support.MatchContextForTests(str, start, end, self.flags)
-
-def _bytepos_to_charindex(self, bytepos):
-    if isinstance(self.ctx, support.MatchContextForTests):
-        return self.ctx._real_pos(bytepos)
-    return _org_maker[1](self, bytepos)
-
-def setup_module(mod):
-    mod._org_maker = (
-        interp_sre.W_SRE_Pattern._make_str_match_context,
-        interp_sre.W_SRE_Match.bytepos_to_charindex,
-        )
-    interp_sre.W_SRE_Pattern._make_str_match_context = _test_sre_ctx_
-    interp_sre.W_SRE_Match.bytepos_to_charindex = _bytepos_to_charindex
-
-def teardown_module(mod):
-    (
-        interp_sre.W_SRE_Pattern._make_str_match_context,
-        interp_sre.W_SRE_Match.bytepos_to_charindex,
-    ) = mod._org_maker
-
 
 class AppTestSrePy:
     def test_magic(self):
@@ -149,9 +116,6 @@
         assert ['', 'a', 'l', 'a', 'lla'] == re.split("b(a)", "balballa")
         assert ['', 'a', None, 'l', 'u', None, 'lla'] == (
             re.split("b([ua]|(s))", "balbulla"))
-        assert ["abc"] == re.split("", "abc")
-        assert ["abc"] == re.split("X?", "abc")
-        assert ["a", "c"] == re.split("b?", "abc")
 
     def test_weakref(self):
         import re, _weakref
@@ -285,7 +249,6 @@
         assert b"rbd\nbr\n" == re.sub(b"a(.)", br"b\1\n", b"radar")
         assert (b"rbd\nbr\n", 2) == re.subn(b"a(.)", br"b\1\n", b"radar")
         assert (b"bbbba", 2) == re.subn(b"a", b"b", b"ababa", 2)
-        assert "XaXbXcX" == re.sub("", "X", "abc")
 
     def test_sub_unicode(self):
         import re

From pypy.commits at gmail.com  Tue Jul 10 02:19:54 2018
From: pypy.commits at gmail.com (mattip)
Date: Mon, 09 Jul 2018 23:19:54 -0700 (PDT)
Subject: [pypy-commit] pypy default: use State in place of global variable
Message-ID: <5b44500a.1c69fb81.329e4.700f@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r94840:9003631c15df
Date: 2018-07-09 23:14 -0700
http://bitbucket.org/pypy/pypy/changeset/9003631c15df/

Log:	use State in place of global variable

diff --git a/pypy/module/cpyext/cdatetime.py b/pypy/module/cpyext/cdatetime.py
--- a/pypy/module/cpyext/cdatetime.py
+++ b/pypy/module/cpyext/cdatetime.py
@@ -12,18 +12,19 @@
 from pypy.module.__pypy__.interp_pypydatetime import (W_DateTime_Date,
     W_DateTime_Time, W_DateTime_Delta)
 from rpython.tool.sourcetools import func_renamer
+from pypy.module.cpyext.state import State
 
 cts.parse_header(parse_dir / 'cpyext_datetime.h')
 
 
 PyDateTime_CAPI = cts.gettype('PyDateTime_CAPI')
 
-datetimeAPI_global = []
 
 @cpython_api([], lltype.Ptr(PyDateTime_CAPI))
 def _PyDateTime_Import(space):
-    if len(datetimeAPI_global) >0:
-        return datetimeAPI_global[0]
+    state = space.fromcache(State)
+    if len(state.datetimeAPI) > 0:
+        return state.datetimeAPI[0]
     datetimeAPI = lltype.malloc(PyDateTime_CAPI, flavor='raw',
                                 track_allocation=False)
 
@@ -66,8 +67,8 @@
         _PyDelta_FromDelta.api_func.functype,
         _PyDelta_FromDelta.api_func.get_wrapper(space))
 
-    datetimeAPI_global.append(datetimeAPI)
-    return datetimeAPI
+    state.datetimeAPI.append(datetimeAPI)
+    return state.datetimeAPI[0]
 
 PyDateTime_Time = cts.gettype('PyDateTime_Time*')
 PyDateTime_DateTime = cts.gettype('PyDateTime_DateTime*')
@@ -135,8 +136,10 @@
     '''Fills a newly allocated py_obj from the w_obj
     If it is a datetime.time or datetime.datetime, it may have tzinfo
     '''
-    assert len(datetimeAPI_global) > 0
-    if datetimeAPI_global[0].c_TimeType == py_obj.c_ob_type:
+    state = space.fromcache(State)
+    # cannot raise here, so just crash
+    assert len(state.datetimeAPI) > 0
+    if state.datetimeAPI[0].c_TimeType == py_obj.c_ob_type:
         py_datetime = rffi.cast(PyDateTime_Time, py_obj)
         w_tzinfo = space.getattr(w_obj, space.newtext('tzinfo'))
         if space.is_none(w_tzinfo):
@@ -145,7 +148,7 @@
         else:
             py_datetime.c_hastzinfo = cts.cast('unsigned char', 1)
             py_datetime.c_tzinfo = make_ref(space, w_tzinfo)
-    elif datetimeAPI_global[0].c_DateTimeType == py_obj.c_ob_type:
+    elif state.datetimeAPI[0].c_DateTimeType == py_obj.c_ob_type:
         # For now this is exactly the same structure as PyDateTime_Time
         py_datetime = rffi.cast(PyDateTime_DateTime, py_obj)
         w_tzinfo = space.getattr(w_obj, space.newtext('tzinfo'))
@@ -159,12 +162,14 @@
 @slot_function([PyObject], lltype.Void)
 def type_dealloc(space, py_obj):
     from pypy.module.cpyext.object import _dealloc
-    assert len(datetimeAPI_global) > 0
-    if datetimeAPI_global[0].c_TimeType == py_obj.c_ob_type:
+    state = space.fromcache(State)
+    # cannot raise here, so just crash
+    assert len(state.datetimeAPI) > 0
+    if state.datetimeAPI[0].c_TimeType == py_obj.c_ob_type:
         py_datetime = rffi.cast(PyDateTime_Time, py_obj)
         if (widen(py_datetime.c_hastzinfo) != 0):
             decref(space, py_datetime.c_tzinfo)
-    elif datetimeAPI_global[0].c_DateTimeType == py_obj.c_ob_type:
+    elif state.datetimeAPI[0].c_DateTimeType == py_obj.c_ob_type:
         py_datetime = rffi.cast(PyDateTime_DateTime, py_obj)
         if (widen(py_datetime.c_hastzinfo) != 0):
             decref(space, py_datetime.c_tzinfo)
diff --git a/pypy/module/cpyext/state.py b/pypy/module/cpyext/state.py
--- a/pypy/module/cpyext/state.py
+++ b/pypy/module/cpyext/state.py
@@ -42,6 +42,8 @@
         # A mapping {filename: copy-of-the-w_dict}, similar to CPython's
         # variable 'extensions' in Python/import.c.
         self.extensions = {}
+        # XXX will leak if _PyDateTime_Import already called
+        self.datetimeAPI = []
 
     def set_exception(self, operror):
         self.clear_exception()

From pypy.commits at gmail.com  Tue Jul 10 02:27:26 2018
From: pypy.commits at gmail.com (wlav)
Date: Mon, 09 Jul 2018 23:27:26 -0700 (PDT)
Subject: [pypy-commit] pypy cppyy-packaging: cosmetics
Message-ID: <5b4451ce.1c69fb81.7fcaf.db23@mx.google.com>

Author: Wim Lavrijsen <WLavrijsen at lbl.gov>
Branch: cppyy-packaging
Changeset: r94841:814abb166218
Date: 2018-07-09 21:57 -0700
http://bitbucket.org/pypy/pypy/changeset/814abb166218/

Log:	cosmetics

diff --git a/pypy/module/_cppyy/interp_cppyy.py b/pypy/module/_cppyy/interp_cppyy.py
--- a/pypy/module/_cppyy/interp_cppyy.py
+++ b/pypy/module/_cppyy/interp_cppyy.py
@@ -258,7 +258,8 @@
         jit.promote(self)
         cif_descr = self.cif_descr
         # add extra space for const-ref support (see converter.py)
-        buffer = lltype.malloc(rffi.CCHARP.TO, cif_descr.exchange_size+len(self.arg_defs)*rffi.sizeof(rffi.DOUBLE), flavor='raw')
+        buffer = lltype.malloc(rffi.CCHARP.TO,
+            cif_descr.exchange_size+len(self.arg_defs)*rffi.sizeof(rffi.DOUBLE), flavor='raw')
         thisoff = 0
         try:
             if cppthis:
@@ -632,11 +633,7 @@
     @unwrap_spec(args_w='args_w')
     def call_args(self, args_w):
         jit.promote(self)
-        #if isinstance(args_w[0], W_CPPInstance):
-            # free function used as bound method, leave in place
-        return self.call_impl(capi.C_NULL_OBJECT, args_w)
-        # free functions are implemented as methods of 'namespace' classes, remove 'instance'
-        #return self.call_impl(capi.C_NULL_OBJECT, args_w[1:])
+        return self.call_impl(capi.C_NULL_OBJECT, args_w, 0)
 
     def __repr__(self):
         return "W_CPPStaticOverload(%s)" % [f.prototype() for f in self.functions]
@@ -1320,7 +1317,7 @@
     def get_base_offset(self, cppinstance, calling_scope):
         assert self == cppinstance.clsdecl
         offset = capi.c_base_offset(self.space,
-                                    self, calling_scope, cppinstance.get_rawobject(), 1)
+            self, calling_scope, cppinstance.get_rawobject(), 1)
         return offset
 
     def get_cppthis(self, cppinstance, calling_scope):

From pypy.commits at gmail.com  Tue Jul 10 02:27:28 2018
From: pypy.commits at gmail.com (wlav)
Date: Mon, 09 Jul 2018 23:27:28 -0700 (PDT)
Subject: [pypy-commit] pypy cppyy-packaging: minor performance improvement
Message-ID: <5b4451d0.1c69fb81.640c2.6305@mx.google.com>

Author: Wim Lavrijsen <WLavrijsen at lbl.gov>
Branch: cppyy-packaging
Changeset: r94842:d69a3855cc3a
Date: 2018-07-09 23:07 -0700
http://bitbucket.org/pypy/pypy/changeset/d69a3855cc3a/

Log:	minor performance improvement

diff --git a/pypy/module/_cppyy/converter.py b/pypy/module/_cppyy/converter.py
--- a/pypy/module/_cppyy/converter.py
+++ b/pypy/module/_cppyy/converter.py
@@ -491,7 +491,7 @@
         from pypy.module._cppyy.interp_cppyy import W_CPPInstance
         if isinstance(w_obj, W_CPPInstance):
             from pypy.module._cppyy.interp_cppyy import INSTANCE_FLAGS_IS_RVALUE
-            if w_obj.flags & INSTANCE_FLAGS_IS_RVALUE:
+            if w_obj.rt_flags & INSTANCE_FLAGS_IS_RVALUE:
                 # reject moves as all are explicit
                 raise ValueError("lvalue expected")
             if capi.c_is_subtype(space, w_obj.clsdecl, self.clsdecl):
@@ -523,14 +523,14 @@
         from pypy.module._cppyy.interp_cppyy import W_CPPInstance, INSTANCE_FLAGS_IS_RVALUE
         obj = space.interp_w(W_CPPInstance, w_obj)
         if obj:
-            if obj.flags & INSTANCE_FLAGS_IS_RVALUE:
-                obj.flags &= ~INSTANCE_FLAGS_IS_RVALUE
+            if obj.rt_flags & INSTANCE_FLAGS_IS_RVALUE:
+                obj.rt_flags &= ~INSTANCE_FLAGS_IS_RVALUE
                 try:
                     return InstanceRefConverter._unwrap_object(self, space, w_obj)
                 except Exception:
                     # TODO: if the method fails on some other converter, then the next
                     # overload can not be an rvalue anymore
-                    obj.flags |= INSTANCE_FLAGS_IS_RVALUE
+                    obj.rt_flags |= INSTANCE_FLAGS_IS_RVALUE
                     raise
         raise oefmt(space.w_ValueError, "object is not an rvalue")
 
diff --git a/pypy/module/_cppyy/interp_cppyy.py b/pypy/module/_cppyy/interp_cppyy.py
--- a/pypy/module/_cppyy/interp_cppyy.py
+++ b/pypy/module/_cppyy/interp_cppyy.py
@@ -633,7 +633,7 @@
     @unwrap_spec(args_w='args_w')
     def call_args(self, args_w):
         jit.promote(self)
-        return self.call_impl(capi.C_NULL_OBJECT, args_w, 0)
+        return self.call_impl(capi.C_NULL_OBJECT, args_w)
 
     def __repr__(self):
         return "W_CPPStaticOverload(%s)" % [f.prototype() for f in self.functions]
@@ -699,7 +699,7 @@
                     if args_w:
                         # try to specialize the type match for the given object
                         cppinstance = self.space.interp_w(W_CPPInstance, args_w[i])
-                        if cppinstance.flags & INSTANCE_FLAGS_IS_RVALUE:
+                        if cppinstance.rt_flags & INSTANCE_FLAGS_IS_RVALUE:
                             sugar = "&&"
                         elif cppinstance.flags & INSTANCE_FLAGS_IS_REF:
                             sugar = "*"
@@ -1340,9 +1340,9 @@
 
 
 class W_CPPInstance(W_Root):
-    _attrs_ = ['space', 'clsdecl', '_rawobject', 'smartdecl', 'deref', 'flags',
+    _attrs_ = ['space', 'clsdecl', '_rawobject', 'smartdecl', 'deref', 'flags', 'rt_flags',
                'finalizer_registered']
-    _immutable_fields_ = ['clsdecl', 'smartdecl', 'deref']
+    _immutable_fields_ = ['clsdecl', 'smartdecl', 'deref', 'flags']
 
     finalizer_registered = False
 
@@ -1357,15 +1357,16 @@
         self.flags = 0
         if isref or (smartdecl and deref):
             self.flags |= INSTANCE_FLAGS_IS_REF
+        self.rt_flags = 0
         if python_owns:
-            self.flags |= INSTANCE_FLAGS_PYTHON_OWNS
+            self.rt_flags |= INSTANCE_FLAGS_PYTHON_OWNS
             self._opt_register_finalizer()
         self.smartdecl = smartdecl
         self.deref     = deref
 
     def _opt_register_finalizer(self):
         if not self.finalizer_registered and not hasattr(self.space, "fake"):
-            assert self.flags & INSTANCE_FLAGS_PYTHON_OWNS
+            assert self.rt_flags & INSTANCE_FLAGS_PYTHON_OWNS
             self.register_finalizer(self.space)
             self.finalizer_registered = True
 
@@ -1377,15 +1378,15 @@
 
     # allow user to determine ownership rules on a per object level
     def fget_python_owns(self, space):
-        return space.newbool(bool(self.flags & INSTANCE_FLAGS_PYTHON_OWNS))
+        return space.newbool(bool(self.rt_flags & INSTANCE_FLAGS_PYTHON_OWNS))
 
     @unwrap_spec(value=bool)
     def fset_python_owns(self, space, value):
         if space.is_true(value):
-            self.flags |= INSTANCE_FLAGS_PYTHON_OWNS
+            self.rt_flags |= INSTANCE_FLAGS_PYTHON_OWNS
             self._opt_register_finalizer()
         else:
-            self.flags &= ~INSTANCE_FLAGS_PYTHON_OWNS
+            self.rt_flags &= ~INSTANCE_FLAGS_PYTHON_OWNS
 
     def get_cppthis(self, calling_scope):
         return self.clsdecl.get_cppthis(self, calling_scope)
@@ -1505,7 +1506,7 @@
             self._rawobject = capi.C_NULL_OBJECT
 
     def _finalize_(self):
-        if self.flags & INSTANCE_FLAGS_PYTHON_OWNS:
+        if self.rt_flags & INSTANCE_FLAGS_PYTHON_OWNS:
             self.destruct()
 
 W_CPPInstance.typedef = TypeDef(
@@ -1649,7 +1650,7 @@
     """Casts the given instance into an C++-style rvalue."""
     obj = space.interp_w(W_CPPInstance, w_obj)
     if obj:
-        obj.flags |= INSTANCE_FLAGS_IS_RVALUE
+        obj.rt_flags |= INSTANCE_FLAGS_IS_RVALUE
     return w_obj
 
 

From pypy.commits at gmail.com  Tue Jul 10 10:21:24 2018
From: pypy.commits at gmail.com (arigo)
Date: Tue, 10 Jul 2018 07:21:24 -0700 (PDT)
Subject: [pypy-commit] cffi default: no real point in turning this warning
 into an error
Message-ID: <5b44c0e4.1c69fb81.63bb5.ed39@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r3128:b3b6b86cb30c
Date: 2018-07-10 16:16 +0200
http://bitbucket.org/cffi/cffi/changeset/b3b6b86cb30c/

Log:	no real point in turning this warning into an error

diff --git a/c/test_c.py b/c/test_c.py
--- a/c/test_c.py
+++ b/c/test_c.py
@@ -3946,8 +3946,8 @@
 
 def test_char_pointer_conversion():
     import warnings
-    assert __version__.startswith(("1.8", "1.9", "1.10", "1.11", "1.12")), (
-        "consider turning the warning into an error")
+    assert __version__.startswith("1."), (
+        "the warning will be an error if we ever release cffi 2.x")
     BCharP = new_pointer_type(new_primitive_type("char"))
     BIntP = new_pointer_type(new_primitive_type("int"))
     BVoidP = new_pointer_type(new_void_type())

From pypy.commits at gmail.com  Tue Jul 10 10:21:26 2018
From: pypy.commits at gmail.com (arigo)
Date: Tue, 10 Jul 2018 07:21:26 -0700 (PDT)
Subject: [pypy-commit] cffi default: setup.py now supports building in a
 context and installing in a different
Message-ID: <5b44c0e6.1c69fb81.a0a19.9045@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r3129:68933f0ea99a
Date: 2018-07-10 16:20 +0200
http://bitbucket.org/cffi/cffi/changeset/68933f0ea99a/

Log:	setup.py now supports building in a context and installing in a
	different context with no compiler. You still get the error message
	about no compiler found, but now the error is not fatal.

diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -46,6 +46,7 @@
             #
             resultlist[:] = res
 
+no_compiler_found = False
 def no_working_compiler_found():
     sys.stderr.write("""
     No working compiler found, or bogus compiler options passed to
@@ -55,8 +56,13 @@
     tries to compile C code.  (Hints: on OS/X 10.8, for errors about
     -mno-fused-madd see http://stackoverflow.com/questions/22313407/
     Otherwise, see https://wiki.python.org/moin/CompLangPython or
-    the IRC channel #python on irc.freenode.net.)\n""")
-    sys.exit(1)
+    the IRC channel #python on irc.freenode.net.)
+
+    Trying to continue anyway.  If you are trying to install CFFI from
+    a build done in a different context, you can ignore this warning.
+    \n""")
+    global no_compiler_found
+    no_compiler_found = True
 
 def get_config():
     from distutils.core import Distribution
@@ -75,11 +81,12 @@
         ok1 = config.try_compile('int some_regular_variable_42;')
         if not ok1:
             no_working_compiler_found()
-        sys.stderr.write("Note: will not use '__thread' in the C code\n")
-        _safe_to_ignore()
+        else:
+            sys.stderr.write("Note: will not use '__thread' in the C code\n")
+            _safe_to_ignore()
 
 def ask_supports_sync_synchronize():
-    if sys.platform == 'win32':
+    if sys.platform == 'win32' or no_compiler_found:
         return
     config = get_config()
     ok = config.try_link('int main(void) { __sync_synchronize(); return 0; }')

From pypy.commits at gmail.com  Wed Jul 11 09:53:52 2018
From: pypy.commits at gmail.com (mattip)
Date: Wed, 11 Jul 2018 06:53:52 -0700 (PDT)
Subject: [pypy-commit] pypy unicode-utf8-py3: str_w -> text_w
Message-ID: <5b460bf0.1c69fb81.d77db.fc50@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: unicode-utf8-py3
Changeset: r94843:0c716461ad5d
Date: 2018-07-10 21:48 -0700
http://bitbucket.org/pypy/pypy/changeset/0c716461ad5d/

Log:	str_w -> text_w

diff --git a/pypy/objspace/std/test/test_bytesobject.py b/pypy/objspace/std/test/test_bytesobject.py
--- a/pypy/objspace/std/test/test_bytesobject.py
+++ b/pypy/objspace/std/test/test_bytesobject.py
@@ -97,7 +97,7 @@
         monkeypatch.setattr(jit, 'isconstant', lambda x: True)
         space = self.space
         w_res = space.call_function(space.w_bytes, space.wrap([42]))
-        assert space.str_w(w_res) == '*'
+        assert space.text_w(w_res) == '*'
 
 
 class AppTestBytesObject:
diff --git a/pypy/objspace/std/test/test_liststrategies.py b/pypy/objspace/std/test/test_liststrategies.py
--- a/pypy/objspace/std/test/test_liststrategies.py
+++ b/pypy/objspace/std/test/test_liststrategies.py
@@ -616,7 +616,7 @@
         space = self.space
         w_l = self.space.newlist([self.space.wrap('a'), self.space.wrap('b')])
         w_l.getitems = None
-        assert space.str_w(space.call_method(space.wrap("c"), "join", w_l)) == "acb"
+        assert space.text_w(space.call_method(space.wrap("c"), "join", w_l)) == "acb"
         #
         # the same for unicode
         w_l = self.space.newlist([self.space.wrap(u'a'), self.space.wrap(u'b')])
diff --git a/pypy/objspace/std/test/test_mapdict.py b/pypy/objspace/std/test/test_mapdict.py
--- a/pypy/objspace/std/test/test_mapdict.py
+++ b/pypy/objspace/std/test/test_mapdict.py
@@ -920,7 +920,7 @@
         #
         def check(space, w_func, name):
             w_code = space.getattr(w_func, space.wrap('__code__'))
-            nameindex = map(space.str_w, w_code.co_names_w).index(name)
+            nameindex = map(space.text_w, w_code.co_names_w).index(name)
             entry = w_code._mapdict_caches[nameindex]
             entry.failure_counter = 0
             entry.success_counter = 0
diff --git a/pypy/objspace/std/test/test_memoryobject.py b/pypy/objspace/std/test/test_memoryobject.py
--- a/pypy/objspace/std/test/test_memoryobject.py
+++ b/pypy/objspace/std/test/test_memoryobject.py
@@ -307,7 +307,7 @@
         self.w_arr = w_arr
         self.arr = []
         self.ndim = space.int_w(w_dim)
-        self.format = space.str_w(w_fmt)
+        self.format = space.text_w(w_fmt)
         self.itemsize = space.int_w(w_itemsize)
         self.strides = []
         for w_i in w_strides.getitems_unroll():
diff --git a/pypy/objspace/std/test/test_methodcache.py b/pypy/objspace/std/test/test_methodcache.py
--- a/pypy/objspace/std/test/test_methodcache.py
+++ b/pypy/objspace/std/test/test_methodcache.py
@@ -52,7 +52,7 @@
         #             if cached_name is name:
 
         # in py3k, identifiers are stored in W_UnicodeObject and unwrapped by
-        # calling space.str_w, which .encode('ascii') the string, thus
+        # calling space.text_w, which .encode('ascii') the string, thus
         # creating new strings all the time. The problem should be solved when
         # we implement proper unicode identifiers in py3k
         @self.retry
diff --git a/pypy/objspace/std/test/test_setobject.py b/pypy/objspace/std/test/test_setobject.py
--- a/pypy/objspace/std/test/test_setobject.py
+++ b/pypy/objspace/std/test/test_setobject.py
@@ -49,9 +49,9 @@
 
     def test_space_newset(self):
         s = self.space.newset()
-        assert self.space.str_w(self.space.repr(s)) == 'set()'
+        assert self.space.text_w(self.space.repr(s)) == 'set()'
         # check that the second time we don't get 'set(...)'
-        assert self.space.str_w(self.space.repr(s)) == 'set()'
+        assert self.space.text_w(self.space.repr(s)) == 'set()'
 
     def test_intersection_order(self):
         # theses tests make sure that intersection is done in the correct order
diff --git a/pypy/objspace/std/test/test_stdobjspace.py b/pypy/objspace/std/test/test_stdobjspace.py
--- a/pypy/objspace/std/test/test_stdobjspace.py
+++ b/pypy/objspace/std/test/test_stdobjspace.py
@@ -16,9 +16,9 @@
         assert self.space.isinstance_w(self.space.newtext("abc"), self.space.w_unicode)
         assert self.space.eq_w(self.space.newtext("&#252;&#246;&#228;"), self.space.newtext(u"&#252;&#246;&#228;"))
 
-    def test_str_w_non_str(self):
-        raises(OperationError,self.space.str_w,self.space.wrap(None))
-        raises(OperationError,self.space.str_w,self.space.wrap(0))
+    def test_text_w_non_str(self):
+        raises(OperationError,self.space.text_w,self.space.wrap(None))
+        raises(OperationError,self.space.text_w,self.space.wrap(0))
 
     def test_int_w_non_int(self):
         raises(OperationError,self.space.int_w,self.space.wrap(None))

From pypy.commits at gmail.com  Wed Jul 11 09:53:55 2018
From: pypy.commits at gmail.com (mattip)
Date: Wed, 11 Jul 2018 06:53:55 -0700 (PDT)
Subject: [pypy-commit] pypy unicode-utf8-py3: revert some of 2621c0f70d91,
 avoid decoding utf8 -> unicode
Message-ID: <5b460bf3.1c69fb81.41cb0.4ba9@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: unicode-utf8-py3
Changeset: r94844:ad7718874721
Date: 2018-07-11 06:44 -0700
http://bitbucket.org/pypy/pypy/changeset/ad7718874721/

Log:	revert some of 2621c0f70d91, avoid decoding utf8 -> unicode

diff --git a/pypy/interpreter/argument.py b/pypy/interpreter/argument.py
--- a/pypy/interpreter/argument.py
+++ b/pypy/interpreter/argument.py
@@ -602,8 +602,11 @@
 
     def getmsg(self):
         if self.num_kwds == 1:
-            msg = u"got an unexpected keyword argument '%s'" % (
-                self.kwd_name.decode('utf8'))
+            if isinstance(self.kwd_name, str):
+                uname = self.kwd_name.decode('utf8')
+            else:
+                uname = self.kwd_name
+            msg = u"got an unexpected keyword argument '%s'" % uname
         else:
             msg = "got %d unexpected keyword arguments" % (
                 self.num_kwds)
diff --git a/pypy/module/_codecs/interp_codecs.py b/pypy/module/_codecs/interp_codecs.py
--- a/pypy/module/_codecs/interp_codecs.py
+++ b/pypy/module/_codecs/interp_codecs.py
@@ -638,7 +638,7 @@
         state = space.fromcache(CodecState)
         utf8len = w_arg._length
         # XXX deal with func() returning length or not
-        result = func(w_arg._utf8.decode('utf8'), errors, state.encode_error_handler)
+        result = func(w_arg._utf8, errors, state.encode_error_handler)
         return space.newtuple([space.newbytes(result.encode('utf8')), space.newint(utf8len)])
     wrap_encoder.__name__ = func.__name__
     globals()[name] = wrap_encoder
diff --git a/pypy/objspace/std/test/test_dictmultiobject.py b/pypy/objspace/std/test/test_dictmultiobject.py
--- a/pypy/objspace/std/test/test_dictmultiobject.py
+++ b/pypy/objspace/std/test/test_dictmultiobject.py
@@ -1290,7 +1290,7 @@
 
     def text_w(self, u):
         assert isinstance(u, unicode)
-        return FakeUnicode(u)
+        return FakeUnicode(u).encode('utf8')
 
     def bytes_w(self, string):
         assert isinstance(string, str)

From pypy.commits at gmail.com  Wed Jul 11 09:53:58 2018
From: pypy.commits at gmail.com (mattip)
Date: Wed, 11 Jul 2018 06:53:58 -0700 (PDT)
Subject: [pypy-commit] pypy unicode-utf8-py3: modify never used rpython
 surrogate_in_utf8 to return index
Message-ID: <5b460bf6.1c69fb81.7e66c.c87d@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: unicode-utf8-py3
Changeset: r94845:4c9e67d171a9
Date: 2018-07-11 06:46 -0700
http://bitbucket.org/pypy/pypy/changeset/4c9e67d171a9/

Log:	modify never used rpython surrogate_in_utf8 to return index

diff --git a/rpython/rlib/rutf8.py b/rpython/rlib/rutf8.py
--- a/rpython/rlib/rutf8.py
+++ b/rpython/rlib/rutf8.py
@@ -471,8 +471,8 @@
     """
     for i in range(len(value) - 2):
         if value[i] == '\xed' and value[i + 1] >= '\xa0':
-            return True
-    return False
+            return i
+    return -1
 
 
 UTF8_INDEX_STORAGE = lltype.GcArray(lltype.Struct('utf8_loc_elem',
diff --git a/rpython/rlib/test/test_rutf8.py b/rpython/rlib/test/test_rutf8.py
--- a/rpython/rlib/test/test_rutf8.py
+++ b/rpython/rlib/test/test_rutf8.py
@@ -152,7 +152,7 @@
 @example([u'\ud800', u'\udc00'])
 def test_surrogate_in_utf8(unichars):
     uni = ''.join([u.encode('utf8') for u in unichars])
-    result = rutf8.surrogate_in_utf8(uni)
+    result = rutf8.surrogate_in_utf8(uni) < 0
     expected = any(uch for uch in unichars if u'\ud800' <= uch <= u'\udfff')
     assert result == expected
 

From pypy.commits at gmail.com  Wed Jul 11 09:54:13 2018
From: pypy.commits at gmail.com (mattip)
Date: Wed, 11 Jul 2018 06:54:13 -0700 (PDT)
Subject: [pypy-commit] pypy unicode-utf8-py3: surrogate and illegal unicode
 handling
Message-ID: <5b460c05.1c69fb81.86e1a.1659@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: unicode-utf8-py3
Changeset: r94846:9cf4fc74394c
Date: 2018-07-11 06:49 -0700
http://bitbucket.org/pypy/pypy/changeset/9cf4fc74394c/

Log:	surrogate and illegal unicode handling

diff --git a/pypy/interpreter/unicodehelper.py b/pypy/interpreter/unicodehelper.py
--- a/pypy/interpreter/unicodehelper.py
+++ b/pypy/interpreter/unicodehelper.py
@@ -29,11 +29,17 @@
                                              space.newtext(msg)]))
     return raise_unicode_exception_decode
 
+def decode_never_raise(errors, encoding, msg, s, startingpos, endingpos):
+    ux = ['\ux' + hex(ord(x))[2:].upper() for x in s[startingpos:endingpos]]
+    return ''.join(ux), endingpos
+
 @specialize.memo()
 def encode_error_handler(space):
     # Fast version of the "strict" errors handler.
     def raise_unicode_exception_encode(errors, encoding, msg, utf8,
                                        startingpos, endingpos):
+        if isinstance(utf8, unicode):
+            utf8 = utf8.encode('utf8')
         u_len = rutf8.get_utf8_length(utf8)
         raise OperationError(space.w_UnicodeEncodeError,
                              space.newtuple([space.newtext(encoding),
@@ -993,7 +999,7 @@
     # Surrogate-preserving utf-8 decoding.  Assuming there is no
     # encoding error, it should always be reversible, and the reverse is
     # encode_utf8sp().
-    return str_decode_utf8(string, "string", True, decode_error_handler(space),
+    return str_decode_utf8(string, "string", True, decode_never_raise,
                            allow_surrogates=True)
 
 
diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -1187,22 +1187,26 @@
 
 
 def encode_object(space, w_object, encoding, errors):
+    utf8 = space.utf8_w(w_object)
+    idx = rutf8.surrogate_in_utf8(utf8)
+    if idx >= 0:
+        eh = unicodehelper.encode_error_handler(space)
+        eh(None, "utf8", "surrogates not allowed", utf8,
+            idx, idx + 1)
     if errors is None or errors == 'strict':
         if encoding is None or encoding == 'utf-8':
-            utf8 = space.utf8_w(w_object)
-            if rutf8.has_surrogates(utf8):
-                utf8 = rutf8.reencode_utf8_with_surrogates(utf8)
+            #if rutf8.has_surrogates(utf8):
+            #    utf8 = rutf8.reencode_utf8_with_surrogates(utf8)
             return space.newbytes(utf8)
         elif encoding == 'ascii':
-            s = space.utf8_w(w_object)
             try:
-                rutf8.check_ascii(s)
+                rutf8.check_ascii(utf8)
             except rutf8.CheckError as a:
                 eh = unicodehelper.encode_error_handler(space)
-                eh(None, "ascii", "ordinal not in range(128)", s,
+                eh(None, "ascii", "ordinal not in range(128)", utf8,
                     a.pos, a.pos + 1)
                 assert False, "always raises"
-            return space.newbytes(s)
+            return space.newbytes(utf8)
 
     from pypy.module._codecs.interp_codecs import encode_text
     if encoding is None:

From pypy.commits at gmail.com  Wed Jul 11 09:54:15 2018
From: pypy.commits at gmail.com (mattip)
Date: Wed, 11 Jul 2018 06:54:15 -0700 (PDT)
Subject: [pypy-commit] pypy unicode-utf8-py3: small tweaks,
 copy partition from py3.5
Message-ID: <5b460c07.1c69fb81.a86ff.58ec@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: unicode-utf8-py3
Changeset: r94847:e8413e4a1934
Date: 2018-07-11 06:51 -0700
http://bitbucket.org/pypy/pypy/changeset/e8413e4a1934/

Log:	small tweaks, copy partition from py3.5

diff --git a/pypy/objspace/std/bytesobject.py b/pypy/objspace/std/bytesobject.py
--- a/pypy/objspace/std/bytesobject.py
+++ b/pypy/objspace/std/bytesobject.py
@@ -404,9 +404,9 @@
     _KIND1 = "byte"
     _KIND2 = "bytes"
 
-    def __init__(self, str):
-        assert str is not None
-        self._value = str
+    def __init__(self, s):
+        assert s is not None
+        self._value = s
 
     def __repr__(self):
         """representation for debugging purposes"""
diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py
--- a/pypy/objspace/std/objspace.py
+++ b/pypy/objspace/std/objspace.py
@@ -327,7 +327,7 @@
         return W_ListObject.newlist_bytes(self, list_s)
 
     def newlist_text(self, list_t):
-        return self.newlist_utf8([decode_utf8sp(self, s) for s in list_t])
+        return self.newlist_utf8([decode_utf8sp(self, s)[0] for s in list_t])
 
     def newlist_utf8(self, list_u, is_ascii=True):
         # TODO ignoring is_ascii, is that correct?
@@ -762,7 +762,7 @@
                     if not e.match(self, self.w_TypeError):
                         raise
                 else:
-                    classname = b'%s.%s' % (modulename, classname)
+                    classname = u'%s.%s' % (modulename.decode('utf8'), classname)
         else:
             classname = w_type.name.decode('utf-8')
         return classname
diff --git a/pypy/objspace/std/stringmethods.py b/pypy/objspace/std/stringmethods.py
--- a/pypy/objspace/std/stringmethods.py
+++ b/pypy/objspace/std/stringmethods.py
@@ -478,39 +478,56 @@
         from pypy.objspace.std.bytearrayobject import W_BytearrayObject
         value = self._val(space)
 
-        sub = _get_buffer(space, w_sub)
-        sublen = sub.getlength()
-        if sublen == 0:
-            raise oefmt(space.w_ValueError, "empty separator")
+        if self._use_rstr_ops(space, w_sub):
+            sub = self._op_val(space, w_sub)
+            sublen = len(sub)
+            if sublen == 0:
+                raise oefmt(space.w_ValueError, "empty separator")
 
-        pos = find(value, sub, 0, len(value))
-        if pos != -1 and isinstance(self, W_BytearrayObject):
-            w_sub = self._new_from_buffer(sub)
+            pos = value.find(sub)
+        else:
+            sub = space.readbuf_w(w_sub)
+            sublen = sub.getlength()
+            if sublen == 0:
+                raise oefmt(space.w_ValueError, "empty separator")
+
+            pos = find(value, sub, 0, len(value))
+            if pos != -1 and isinstance(self, W_BytearrayObject):
+                w_sub = self._new_from_buffer(sub)
 
         if pos == -1:
-            self = self._new(value)
+            if isinstance(self, W_BytearrayObject):
+                self = self._new(value)
             return space.newtuple([self, self._empty(), self._empty()])
         else:
             return space.newtuple(
                 [self._sliced(space, value, 0, pos, self), w_sub,
                  self._sliced(space, value, pos + sublen, len(value), self)])
 
-    # This is not used for W_UnicodeObject.
     def descr_rpartition(self, space, w_sub):
         from pypy.objspace.std.bytearrayobject import W_BytearrayObject
         value = self._val(space)
 
-        sub = _get_buffer(space, w_sub)
-        sublen = sub.getlength()
-        if sublen == 0:
-            raise oefmt(space.w_ValueError, "empty separator")
+        if self._use_rstr_ops(space, w_sub):
+            sub = self._op_val(space, w_sub)
+            sublen = len(sub)
+            if sublen == 0:
+                raise oefmt(space.w_ValueError, "empty separator")
 
-        pos = rfind(value, sub, 0, len(value))
-        if pos != -1 and isinstance(self, W_BytearrayObject):
-            w_sub = self._new_from_buffer(sub)
+            pos = value.rfind(sub)
+        else:
+            sub = space.readbuf_w(w_sub)
+            sublen = sub.getlength()
+            if sublen == 0:
+                raise oefmt(space.w_ValueError, "empty separator")
+
+            pos = rfind(value, sub, 0, len(value))
+            if pos != -1 and isinstance(self, W_BytearrayObject):
+                w_sub = self._new_from_buffer(sub)
 
         if pos == -1:
-            self = self._new(value)
+            if isinstance(self, W_BytearrayObject):
+                self = self._new(value)
             return space.newtuple([self._empty(), self._empty(), self])
         else:
             return space.newtuple(
diff --git a/pypy/objspace/std/test/test_stdobjspace.py b/pypy/objspace/std/test/test_stdobjspace.py
--- a/pypy/objspace/std/test/test_stdobjspace.py
+++ b/pypy/objspace/std/test/test_stdobjspace.py
@@ -84,7 +84,7 @@
         from pypy.objspace.std.unicodeobject import W_UnicodeObject
         w_x = self.space.wrap('foo')
         assert isinstance(w_x, W_UnicodeObject)
-        assert w_x._value == u'foo'
+        assert w_x._utf8 == 'foo'
         #
         # calling space.wrap() on a byte string which is not ASCII should
         # never happen. Howeven it might happen while the py3k port is not
@@ -93,4 +93,4 @@
         from pypy.objspace.std.unicodeobject import W_UnicodeObject
         w_x = self.space.wrap('foo\xF0')
         assert isinstance(w_x, W_UnicodeObject)
-        assert w_x._value == u'foo\ufffd'
+        assert w_x._utf8 == 'foo\uxF0'

From pypy.commits at gmail.com  Wed Jul 11 09:54:17 2018
From: pypy.commits at gmail.com (mattip)
Date: Wed, 11 Jul 2018 06:54:17 -0700 (PDT)
Subject: [pypy-commit] pypy unicode-utf8: modify never used rpython
 surrogate_in_utf8 to return index
Message-ID: <5b460c09.1c69fb81.57d1f.0115@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: unicode-utf8
Changeset: r94848:6ebf80250b71
Date: 2018-07-11 06:46 -0700
http://bitbucket.org/pypy/pypy/changeset/6ebf80250b71/

Log:	modify never used rpython surrogate_in_utf8 to return index

diff --git a/rpython/rlib/rutf8.py b/rpython/rlib/rutf8.py
--- a/rpython/rlib/rutf8.py
+++ b/rpython/rlib/rutf8.py
@@ -471,8 +471,8 @@
     """
     for i in range(len(value) - 2):
         if value[i] == '\xed' and value[i + 1] >= '\xa0':
-            return True
-    return False
+            return i
+    return -1
 
 
 UTF8_INDEX_STORAGE = lltype.GcArray(lltype.Struct('utf8_loc_elem',
diff --git a/rpython/rlib/test/test_rutf8.py b/rpython/rlib/test/test_rutf8.py
--- a/rpython/rlib/test/test_rutf8.py
+++ b/rpython/rlib/test/test_rutf8.py
@@ -152,7 +152,7 @@
 @example([u'\ud800', u'\udc00'])
 def test_surrogate_in_utf8(unichars):
     uni = ''.join([u.encode('utf8') for u in unichars])
-    result = rutf8.surrogate_in_utf8(uni)
+    result = rutf8.surrogate_in_utf8(uni) < 0
     expected = any(uch for uch in unichars if u'\ud800' <= uch <= u'\udfff')
     assert result == expected
 

From pypy.commits at gmail.com  Wed Jul 11 10:22:13 2018
From: pypy.commits at gmail.com (mattip)
Date: Wed, 11 Jul 2018 07:22:13 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: merge default into py3.5
Message-ID: <5b461295.1c69fb81.8381.2b98@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: py3.5
Changeset: r94849:d367d9785b64
Date: 2018-07-11 09:20 -0500
http://bitbucket.org/pypy/pypy/changeset/d367d9785b64/

Log:	merge default into py3.5

diff --git a/lib_pypy/cffi.egg-info/PKG-INFO b/lib_pypy/cffi.egg-info/PKG-INFO
--- a/lib_pypy/cffi.egg-info/PKG-INFO
+++ b/lib_pypy/cffi.egg-info/PKG-INFO
@@ -1,6 +1,6 @@
 Metadata-Version: 1.1
 Name: cffi
-Version: 1.11.5
+Version: 1.12.0
 Summary: Foreign Function Interface for Python calling C code.
 Home-page: http://cffi.readthedocs.org
 Author: Armin Rigo, Maciej Fijalkowski
diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py
--- a/lib_pypy/cffi/__init__.py
+++ b/lib_pypy/cffi/__init__.py
@@ -4,8 +4,8 @@
 from .api import FFI
 from .error import CDefError, FFIError, VerificationError, VerificationMissing
 
-__version__ = "1.11.5"
-__version_info__ = (1, 11, 5)
+__version__ = "1.12.0"
+__version_info__ = (1, 12, 0)
 
 # The verifier module file names are based on the CRC32 of a string that
 # contains the following version number.  It may be older than __version__
diff --git a/lib_pypy/cffi/_embedding.h b/lib_pypy/cffi/_embedding.h
--- a/lib_pypy/cffi/_embedding.h
+++ b/lib_pypy/cffi/_embedding.h
@@ -221,7 +221,7 @@
 
         if (f != NULL && f != Py_None) {
             PyFile_WriteString("\nFrom: " _CFFI_MODULE_NAME
-                               "\ncompiled with cffi version: 1.11.5"
+                               "\ncompiled with cffi version: 1.12.0"
                                "\n_cffi_backend module: ", f);
             modules = PyImport_GetModuleDict();
             mod = PyDict_GetItemString(modules, "_cffi_backend");
diff --git a/lib_pypy/cffi/api.py b/lib_pypy/cffi/api.py
--- a/lib_pypy/cffi/api.py
+++ b/lib_pypy/cffi/api.py
@@ -96,18 +96,21 @@
             self.CData, self.CType = backend._get_types()
         self.buffer = backend.buffer
 
-    def cdef(self, csource, override=False, packed=False):
+    def cdef(self, csource, override=False, packed=False, pack=None):
         """Parse the given C source.  This registers all declared functions,
         types, and global variables.  The functions and global variables can
         then be accessed via either 'ffi.dlopen()' or 'ffi.verify()'.
         The types can be used in 'ffi.new()' and other functions.
         If 'packed' is specified as True, all structs declared inside this
         cdef are packed, i.e. laid out without any field alignment at all.
+        Alternatively, 'pack' can be a small integer, and requests for
+        alignment greater than that are ignored (pack=1 is equivalent to
+        packed=True).
         """
-        self._cdef(csource, override=override, packed=packed)
+        self._cdef(csource, override=override, packed=packed, pack=pack)
 
-    def embedding_api(self, csource, packed=False):
-        self._cdef(csource, packed=packed, dllexport=True)
+    def embedding_api(self, csource, packed=False, pack=None):
+        self._cdef(csource, packed=packed, pack=pack, dllexport=True)
         if self._embedding is None:
             self._embedding = ''
 
diff --git a/lib_pypy/cffi/backend_ctypes.py b/lib_pypy/cffi/backend_ctypes.py
--- a/lib_pypy/cffi/backend_ctypes.py
+++ b/lib_pypy/cffi/backend_ctypes.py
@@ -730,7 +730,8 @@
         return self._new_struct_or_union('union', name, ctypes.Union)
 
     def complete_struct_or_union(self, CTypesStructOrUnion, fields, tp,
-                                 totalsize=-1, totalalignment=-1, sflags=0):
+                                 totalsize=-1, totalalignment=-1, sflags=0,
+                                 pack=0):
         if totalsize >= 0 or totalalignment >= 0:
             raise NotImplementedError("the ctypes backend of CFFI does not support "
                                       "structures completed by verify(); please "
@@ -751,6 +752,8 @@
                 bfield_types[fname] = Ellipsis
         if sflags & 8:
             struct_or_union._pack_ = 1
+        elif pack:
+            struct_or_union._pack_ = pack
         struct_or_union._fields_ = cfields
         CTypesStructOrUnion._bfield_types = bfield_types
         #
diff --git a/lib_pypy/cffi/cparser.py b/lib_pypy/cffi/cparser.py
--- a/lib_pypy/cffi/cparser.py
+++ b/lib_pypy/cffi/cparser.py
@@ -306,11 +306,25 @@
             msg = 'parse error\n%s' % (msg,)
         raise CDefError(msg)
 
-    def parse(self, csource, override=False, packed=False, dllexport=False):
+    def parse(self, csource, override=False, packed=False, pack=None,
+                    dllexport=False):
+        if packed:
+            if packed != True:
+                raise ValueError("'packed' should be False or True; use "
+                                 "'pack' to give another value")
+            if pack:
+                raise ValueError("cannot give both 'pack' and 'packed'")
+            pack = 1
+        elif pack:
+            if pack & (pack - 1):
+                raise ValueError("'pack' must be a power of two, not %r" %
+                    (pack,))
+        else:
+            pack = 0
         prev_options = self._options
         try:
             self._options = {'override': override,
-                             'packed': packed,
+                             'packed': pack,
                              'dllexport': dllexport}
             self._internal_parse(csource)
         finally:
diff --git a/lib_pypy/cffi/model.py b/lib_pypy/cffi/model.py
--- a/lib_pypy/cffi/model.py
+++ b/lib_pypy/cffi/model.py
@@ -342,7 +342,7 @@
     fixedlayout = None
     completed = 0
     partial = False
-    packed = False
+    packed = 0
 
     def __init__(self, name, fldnames, fldtypes, fldbitsize, fldquals=None):
         self.name = name
@@ -414,11 +414,14 @@
             fldtypes = [tp.get_cached_btype(ffi, finishlist)
                         for tp in self.fldtypes]
             lst = list(zip(self.fldnames, fldtypes, self.fldbitsize))
-            sflags = 0
+            extra_flags = ()
             if self.packed:
-                sflags = 8    # SF_PACKED
+                if self.packed == 1:
+                    extra_flags = (8,)    # SF_PACKED
+                else:
+                    extra_flags = (0, self.packed)
             ffi._backend.complete_struct_or_union(BType, lst, self,
-                                                  -1, -1, sflags)
+                                                  -1, -1, *extra_flags)
             #
         else:
             fldtypes = []
diff --git a/lib_pypy/cffi/recompiler.py b/lib_pypy/cffi/recompiler.py
--- a/lib_pypy/cffi/recompiler.py
+++ b/lib_pypy/cffi/recompiler.py
@@ -893,6 +893,12 @@
             else:
                 flags.append("_CFFI_F_CHECK_FIELDS")
             if tp.packed:
+                if tp.packed > 1:
+                    raise NotImplementedError(
+                        "%r is declared with 'pack=%r'; only 0 or 1 are "
+                        "supported in API mode (try to use \"...;\", which "
+                        "does not require a 'pack' declaration)" %
+                        (tp, tp.packed))
                 flags.append("_CFFI_F_PACKED")
         else:
             flags.append("_CFFI_F_EXTERNAL")
diff --git a/pypy/doc/config/objspace.disable_entrypoints.txt b/pypy/doc/config/objspace.disable_entrypoints.txt
new file mode 100644
diff --git a/pypy/doc/config/objspace.fstrings.txt b/pypy/doc/config/objspace.fstrings.txt
new file mode 100644
diff --git a/pypy/doc/config/objspace.hash.txt b/pypy/doc/config/objspace.hash.txt
new file mode 100644
diff --git a/pypy/doc/config/objspace.usemodules._frozen_importlib.txt b/pypy/doc/config/objspace.usemodules._frozen_importlib.txt
new file mode 100644
diff --git a/pypy/doc/config/objspace.usemodules._jitlog.txt b/pypy/doc/config/objspace.usemodules._jitlog.txt
new file mode 100644
diff --git a/pypy/doc/config/objspace.usemodules.faulthandler.txt b/pypy/doc/config/objspace.usemodules.faulthandler.txt
new file mode 100644
diff --git a/pypy/doc/config/translation.backendopt.replace_we_are_jitted.txt b/pypy/doc/config/translation.backendopt.replace_we_are_jitted.txt
new file mode 100644
diff --git a/pypy/doc/config/translation.jit_opencoder_model.txt b/pypy/doc/config/translation.jit_opencoder_model.txt
new file mode 100644
diff --git a/pypy/doc/config/translation.keepgoing.txt b/pypy/doc/config/translation.keepgoing.txt
new file mode 100644
diff --git a/pypy/doc/config/translation.libname.txt b/pypy/doc/config/translation.libname.txt
new file mode 100644
diff --git a/pypy/doc/config/translation.lto.txt b/pypy/doc/config/translation.lto.txt
new file mode 100644
diff --git a/pypy/doc/config/translation.profoptargs.txt b/pypy/doc/config/translation.profoptargs.txt
new file mode 100644
diff --git a/pypy/doc/config/translation.reverse_debugger.txt b/pypy/doc/config/translation.reverse_debugger.txt
new file mode 100644
diff --git a/pypy/doc/config/translation.split_gc_address_space.txt b/pypy/doc/config/translation.split_gc_address_space.txt
new file mode 100644
diff --git a/pypy/doc/tool/makecontributor.py b/pypy/doc/tool/makecontributor.py
--- a/pypy/doc/tool/makecontributor.py
+++ b/pypy/doc/tool/makecontributor.py
@@ -18,12 +18,13 @@
     'Antonio Cuni': ['antocuni', 'anto'],
     'Armin Rigo': ['arigo', 'arfigo', 'armin', 'arigato'],
     'Maciej Fijalkowski': ['fijal'],
-    'Carl Friedrich Bolz-Tereick': ['Carl Friedrich Bolz', 'cfbolz', 'cf'],
+    'Carl Friedrich Bolz-Tereick': ['Carl Friedrich Bolz', 'cfbolz', 'cf', 'cbolz'],
     'Samuele Pedroni': ['pedronis', 'samuele', 'samule'],
-    'Richard Plangger':['planrich'],
-    'Michael Hudson': ['mwh'],
+    'Richard Plangger': ['planrich', 'plan_rich'],
+    'Remi Meier': ['remi'],
+    'Michael Hudson-Doyle': ['mwh', 'Michael Hudson'],
     'Holger Krekel': ['hpk', 'holger krekel', 'holger', 'hufpk'],
-    "Amaury Forgeot d'Arc": ['afa'],
+    "Amaury Forgeot d'Arc": ['afa', 'amauryfa at gmail.com'],
     'Alex Gaynor': ['alex', 'agaynor'],
     'David Schneider': ['bivab', 'david'],
     'Christian Tismer': ['chris', 'christian', 'tismer',
@@ -41,7 +42,7 @@
     'Mark Pearse': ['mwp'],
     'Toon Verwaest': ['tverwaes'],
     'Eric van Riet Paap': ['ericvrp'],
-    'Jacob Hallen': ['jacob', 'jakob'],
+    'Jacob Hallen': ['jacob', 'jakob', 'jacob hallen'],
     'Anders Lehmann': ['ale', 'anders'],
     'Bert Freudenberg': ['bert'],
     'Boris Feigin': ['boris', 'boria'],
@@ -69,19 +70,25 @@
     'Manuel Jacob': ['mjacob'],
     'Rami Chowdhury': ['necaris'],
     'Stanislaw Halik': ['Stanislaw Halik', 'w31rd0'],
-    'Wenzhu Man':['wenzhu man', 'wenzhuman'],
-    'Anton Gulenko':['anton gulenko', 'anton_gulenko'],
-    'Richard Lancaster':['richardlancaster'],
-    'William Leslie':['William ML Leslie'],
-    'Spenser Bauman':['Spenser Andrew Bauman'],
-    'Raffael Tfirst':['raffael.tfirst at gmail.com'],
-    'timo':['timo at eistee.fritz.box'],
-    'Jasper Schulz':['Jasper.Schulz', 'jbs'],
-    'Aaron Gallagher':['"Aaron Gallagher'],
-    'Yasir Suhail':['yasirs'],
+    'Wenzhu Man': ['wenzhu man', 'wenzhuman'],
+    'Anton Gulenko': ['anton gulenko', 'anton_gulenko'],
+    'Richard Lancaster': ['richardlancaster'],
+    'William Leslie': ['William ML Leslie'],
+    'Spenser Bauman': ['Spenser Andrew Bauman'],
+    'Raffael Tfirst': ['raffael.tfirst at gmail.com'],
+    'timo': ['timo at eistee.fritz.box'],
+    'Jasper Schulz': ['Jasper.Schulz', 'jbs'],
+    'Aaron Gallagher': ['"Aaron Gallagher'],
+    'Yasir Suhail': ['yasirs'],
     'Squeaky': ['squeaky'],
-    "Amaury Forgeot d'Arc": ['amauryfa at gmail.com'],
     "Dodan Mihai": ['mihai.dodan at gmail.com'],
+    'Wim Lavrijsen': ['wlav'],
+    'Toon Verwaest': ['toon', 'tverwaes'],
+    'Seo Sanghyeon': ['sanxiyn'],
+    'Leonardo Santagada': ['santagada'],
+    'Laurence Tratt': ['ltratt'],
+    'Pieter Zieschang': ['pzieschang', 'p_zieschang at yahoo.de'],
+    'John Witulski': ['witulski'],
     }
 
 alias_map = {}
@@ -103,7 +110,8 @@
         return set()
     ignore_words = ['around', 'consulting', 'yesterday', 'for a bit', 'thanks',
                     'in-progress', 'bits of', 'even a little', 'floating',
-                    'a bit', 'reviewing']
+                    'a bit', 'reviewing', 'looking', 'advising', 'partly', 'ish',
+                    'watching', 'mostly', 'jumping']
     sep_words = ['and', ';', '+', '/', 'with special  by']
     nicknames = match.group(1)
     for word in ignore_words:
diff --git a/pypy/module/_cffi_backend/__init__.py b/pypy/module/_cffi_backend/__init__.py
--- a/pypy/module/_cffi_backend/__init__.py
+++ b/pypy/module/_cffi_backend/__init__.py
@@ -3,7 +3,7 @@
 from rpython.rlib import rdynload, clibffi
 from rpython.rtyper.lltypesystem import rffi
 
-VERSION = "1.11.5"
+VERSION = "1.12.0"
 
 FFI_DEFAULT_ABI = clibffi.FFI_DEFAULT_ABI
 try:
diff --git a/pypy/module/_cffi_backend/newtype.py b/pypy/module/_cffi_backend/newtype.py
--- a/pypy/module/_cffi_backend/newtype.py
+++ b/pypy/module/_cffi_backend/newtype.py
@@ -258,6 +258,11 @@
 SF_PACKED             = 0x08
 SF_STD_FIELD_POS      = 0x80
 
+if sys.platform == 'win32':
+    SF_DEFAULT_PACKING = 8
+else:
+    SF_DEFAULT_PACKING = 0x40000000    # a huge power of two
+
 
 if sys.platform == 'win32':
     DEFAULT_SFLAGS_PLATFORM = SF_MSVC_BITFIELDS
@@ -309,10 +314,18 @@
         w_ctype._custom_field_pos = True
 
 @unwrap_spec(w_ctype=ctypeobj.W_CType, totalsize=int, totalalignment=int,
-             sflags=int)
+             sflags=int, pack=int)
 def complete_struct_or_union(space, w_ctype, w_fields, w_ignored=None,
-                             totalsize=-1, totalalignment=-1, sflags=0):
+                             totalsize=-1, totalalignment=-1, sflags=0,
+                             pack=0):
     sflags = complete_sflags(sflags)
+    if sflags & SF_PACKED:
+        pack = 1
+    elif pack <= 0:
+        pack = SF_DEFAULT_PACKING
+    else:
+        sflags |= SF_PACKED
+
     if (not isinstance(w_ctype, ctypestruct.W_CTypeStructOrUnion)
             or w_ctype.size >= 0):
         raise oefmt(space.w_TypeError,
@@ -362,7 +375,7 @@
         # update the total alignment requirement, but skip it if the
         # field is an anonymous bitfield or if SF_PACKED
         falignorg = ftype.alignof()
-        falign = 1 if sflags & SF_PACKED else falignorg
+        falign = min(pack, falignorg)
         do_align = True
         if (sflags & SF_GCC_ARM_BITFIELDS) == 0 and fbitsize >= 0:
             if (sflags & SF_MSVC_BITFIELDS) == 0:
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -1,7 +1,7 @@
 # ____________________________________________________________
 
 import sys
-assert __version__ == "1.11.5", ("This test_c.py file is for testing a version"
+assert __version__ == "1.12.0", ("This test_c.py file is for testing a version"
                                  " of cffi that differs from the one that we"
                                  " get from 'import _cffi_backend'")
 if sys.version_info < (3,):
@@ -3541,30 +3541,50 @@
     BLong = new_primitive_type("long")
     BChar = new_primitive_type("char")
     BShort = new_primitive_type("short")
-    BStruct = new_struct_type("struct foo")
-    complete_struct_or_union(BStruct, [('a1', BLong, -1),
-                                       ('a2', BChar, -1),
-                                       ('a3', BShort, -1)],
-                             None, -1, -1, SF_PACKED)
-    d = BStruct.fields
-    assert len(d) == 3
-    assert d[0][0] == 'a1'
-    assert d[0][1].type is BLong
+    for extra_args in [(SF_PACKED,), (0, 1)]:
+        BStruct = new_struct_type("struct foo")
+        complete_struct_or_union(BStruct, [('a1', BLong, -1),
+                                           ('a2', BChar, -1),
+                                           ('a3', BShort, -1)],
+                                 None, -1, -1, *extra_args)
+        d = BStruct.fields
+        assert len(d) == 3
+        assert d[0][0] == 'a1'
+        assert d[0][1].type is BLong
+        assert d[0][1].offset == 0
+        assert d[0][1].bitshift == -1
+        assert d[0][1].bitsize == -1
+        assert d[1][0] == 'a2'
+        assert d[1][1].type is BChar
+        assert d[1][1].offset == sizeof(BLong)
+        assert d[1][1].bitshift == -1
+        assert d[1][1].bitsize == -1
+        assert d[2][0] == 'a3'
+        assert d[2][1].type is BShort
+        assert d[2][1].offset == sizeof(BLong) + sizeof(BChar)
+        assert d[2][1].bitshift == -1
+        assert d[2][1].bitsize == -1
+        assert sizeof(BStruct) == sizeof(BLong) + sizeof(BChar) + sizeof(BShort)
+        assert alignof(BStruct) == 1
+    #
+    BStruct2 = new_struct_type("struct foo")
+    complete_struct_or_union(BStruct2, [('b1', BChar, -1),
+                                        ('b2', BLong, -1)],
+                             None, -1, -1, 0, 2)
+    d = BStruct2.fields
+    assert len(d) == 2
+    assert d[0][0] == 'b1'
+    assert d[0][1].type is BChar
     assert d[0][1].offset == 0
     assert d[0][1].bitshift == -1
     assert d[0][1].bitsize == -1
-    assert d[1][0] == 'a2'
-    assert d[1][1].type is BChar
-    assert d[1][1].offset == sizeof(BLong)
+    assert d[1][0] == 'b2'
+    assert d[1][1].type is BLong
+    assert d[1][1].offset == 2
     assert d[1][1].bitshift == -1
     assert d[1][1].bitsize == -1
-    assert d[2][0] == 'a3'
-    assert d[2][1].type is BShort
-    assert d[2][1].offset == sizeof(BLong) + sizeof(BChar)
-    assert d[2][1].bitshift == -1
-    assert d[2][1].bitsize == -1
-    assert sizeof(BStruct) == sizeof(BLong) + sizeof(BChar) + sizeof(BShort)
-    assert alignof(BStruct) == 1
+    assert sizeof(BStruct2) == 2 + sizeof(BLong)
+    assert alignof(BStruct2) == 2
 
 def test_packed_with_bitfields():
     if sys.platform == "win32":
@@ -3915,7 +3935,7 @@
 
 def test_char_pointer_conversion():
     import warnings
-    assert __version__.startswith(("1.8", "1.9", "1.10", "1.11")), (
+    assert __version__.startswith(("1.8", "1.9", "1.10", "1.11", "1.12")), (
         "consider turning the warning into an error")
     BCharP = new_pointer_type(new_primitive_type("char"))
     BIntP = new_pointer_type(new_primitive_type("int"))
diff --git a/pypy/module/cpyext/cdatetime.py b/pypy/module/cpyext/cdatetime.py
--- a/pypy/module/cpyext/cdatetime.py
+++ b/pypy/module/cpyext/cdatetime.py
@@ -120,21 +120,10 @@
                    dealloc=type_dealloc,
                   )
 
-    # why do we need date_dealloc? Since W_DateTime_Date is the base class for
-    # app level datetime.date. If a c-extension class uses datetime.date for its
-    # base class and defines a tp_dealloc, we will get this:
-    # c_class->tp_dealloc == tp_dealloc_func
-    # c_class->tp_base == datetime.date,
-    #                     datetime.date->tp_dealloc = _PyPy_subtype_dealloc
-    # datetime.date->tp_base = W_DateTime_Date
-    #                    W_DateTime_Date->tp_dealloc = _PyPy_subtype_dealloc
-    # but _PyPy_subtype_dealloc will call tp_dealloc_func, which can call its
-    # base's tp_dealloc and we get recursion. So break the recursion by setting
-    # W_DateTime_Date->tp_dealloc
     make_typedescr(W_DateTime_Date.typedef,
                    basestruct=PyDateTime_DateTime.TO,
                    attach=type_attach,
-                   dealloc=date_dealloc,
+                   dealloc=type_dealloc,
                   )
 
     make_typedescr(W_DateTime_Delta.typedef,
@@ -144,30 +133,41 @@
 
 def type_attach(space, py_obj, w_obj, w_userdata=None):
     '''Fills a newly allocated py_obj from the w_obj
+    If it is a datetime.time or datetime.datetime, it may have tzinfo
     '''
-    if space.type(w_obj).name == 'date':
-        # No tzinfo
-        return
-    py_datetime = rffi.cast(PyDateTime_Time, py_obj)
-    w_tzinfo = space.getattr(w_obj, space.newtext('_tzinfo'))
-    if space.is_none(w_tzinfo):
-        py_datetime.c_hastzinfo = cts.cast('unsigned char', 0)
-        py_datetime.c_tzinfo = lltype.nullptr(PyObject.TO)
-    else:
-        py_datetime.c_hastzinfo = cts.cast('unsigned char', 1)
-        py_datetime.c_tzinfo = make_ref(space, w_tzinfo)
+    assert len(datetimeAPI_global) > 0
+    if datetimeAPI_global[0].c_TimeType == py_obj.c_ob_type:
+        py_datetime = rffi.cast(PyDateTime_Time, py_obj)
+        w_tzinfo = space.getattr(w_obj, space.newtext('tzinfo'))
+        if space.is_none(w_tzinfo):
+            py_datetime.c_hastzinfo = cts.cast('unsigned char', 0)
+            py_datetime.c_tzinfo = lltype.nullptr(PyObject.TO)
+        else:
+            py_datetime.c_hastzinfo = cts.cast('unsigned char', 1)
+            py_datetime.c_tzinfo = make_ref(space, w_tzinfo)
+    elif datetimeAPI_global[0].c_DateTimeType == py_obj.c_ob_type:
+        # For now this is exactly the same structure as PyDateTime_Time
+        py_datetime = rffi.cast(PyDateTime_DateTime, py_obj)
+        w_tzinfo = space.getattr(w_obj, space.newtext('tzinfo'))
+        if space.is_none(w_tzinfo):
+            py_datetime.c_hastzinfo = cts.cast('unsigned char', 0)
+            py_datetime.c_tzinfo = lltype.nullptr(PyObject.TO)
+        else:
+            py_datetime.c_hastzinfo = cts.cast('unsigned char', 1)
+            py_datetime.c_tzinfo = make_ref(space, w_tzinfo)
 
 @slot_function([PyObject], lltype.Void)
 def type_dealloc(space, py_obj):
-    py_datetime = rffi.cast(PyDateTime_Time, py_obj)
-    if (widen(py_datetime.c_hastzinfo) != 0):
-        decref(space, py_datetime.c_tzinfo)
     from pypy.module.cpyext.object import _dealloc
-    _dealloc(space, py_obj)
-
- at slot_function([PyObject], lltype.Void)
-def date_dealloc(space, py_obj):
-    from pypy.module.cpyext.object import _dealloc
+    assert len(datetimeAPI_global) > 0
+    if datetimeAPI_global[0].c_TimeType == py_obj.c_ob_type:
+        py_datetime = rffi.cast(PyDateTime_Time, py_obj)
+        if (widen(py_datetime.c_hastzinfo) != 0):
+            decref(space, py_datetime.c_tzinfo)
+    elif datetimeAPI_global[0].c_DateTimeType == py_obj.c_ob_type:
+        py_datetime = rffi.cast(PyDateTime_DateTime, py_obj)
+        if (widen(py_datetime.c_hastzinfo) != 0):
+            decref(space, py_datetime.c_tzinfo)
     _dealloc(space, py_obj)
 
 def timedeltatype_attach(space, py_obj, w_obj, w_userdata=None):
diff --git a/pypy/module/cpyext/test/test_datetime.py b/pypy/module/cpyext/test/test_datetime.py
--- a/pypy/module/cpyext/test/test_datetime.py
+++ b/pypy/module/cpyext/test/test_datetime.py
@@ -307,18 +307,20 @@
         from datetime import tzinfo, datetime, timedelta, time
         # copied from datetime documentation
         class GMT1(tzinfo):
-           def utcoffset(self, dt):
-               return timedelta(hours=1) + self.dst(dt)
-           def dst(self, dt):
-               return timedelta(0)
-           def tzname(self,dt):
+            def __del__(self):
+                print 'deleting GMT1'
+            def utcoffset(self, dt):
+                return timedelta(hours=1) + self.dst(dt)
+            def dst(self, dt):
+                return timedelta(0)
+            def tzname(self,dt):
                 return "GMT +1"
         gmt1 = GMT1()
         dt1 = module.time_with_tzinfo(gmt1)
         assert dt1 == time(6, 6, 6, 6, gmt1)
         assert '+01' in str(dt1)
-        assert module.datetime_with_tzinfo(gmt1) == datetime(
-            2000, 6, 6, 6, 6, 6, 6, gmt1)
+        dt_tz = module.datetime_with_tzinfo(gmt1)
+        assert dt_tz == datetime(2000, 6, 6, 6, 6, 6, 6, gmt1)
 
     def test_checks(self):
         module = self.import_extension('foo', [
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py b/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py
@@ -1824,19 +1824,39 @@
         ffi = FFI(backend=self.Backend())
         ffi.cdef("struct nonpacked { char a; int b; };")
         ffi.cdef("struct is_packed { char a; int b; };", packed=True)
+        ffi.cdef("struct is_packed1 { char a; int b; };", pack=1)
+        ffi.cdef("struct is_packed2 { char a; int b; };", pack=2)
+        ffi.cdef("struct is_packed4 { char a; int b; };", pack=4)
+        ffi.cdef("struct is_packed8 { char a; int b; };", pack=8)
         assert ffi.sizeof("struct nonpacked") == 8
         assert ffi.sizeof("struct is_packed") == 5
+        assert ffi.sizeof("struct is_packed1") == 5
+        assert ffi.sizeof("struct is_packed2") == 6
+        assert ffi.sizeof("struct is_packed4") == 8
+        assert ffi.sizeof("struct is_packed8") == 8
         assert ffi.alignof("struct nonpacked") == 4
         assert ffi.alignof("struct is_packed") == 1
-        s = ffi.new("struct is_packed[2]")
-        s[0].b = 42623381
-        s[0].a = b'X'
-        s[1].b = -4892220
-        s[1].a = b'Y'
-        assert s[0].b == 42623381
-        assert s[0].a == b'X'
-        assert s[1].b == -4892220
-        assert s[1].a == b'Y'
+        assert ffi.alignof("struct is_packed1") == 1
+        assert ffi.alignof("struct is_packed2") == 2
+        assert ffi.alignof("struct is_packed4") == 4
+        assert ffi.alignof("struct is_packed8") == 4
+        for name in ['is_packed', 'is_packed1', 'is_packed2',
+                     'is_packed4', 'is_packed8']:
+            s = ffi.new("struct %s[2]" % name)
+            s[0].b = 42623381
+            s[0].a = b'X'
+            s[1].b = -4892220
+            s[1].a = b'Y'
+            assert s[0].b == 42623381
+            assert s[0].a == b'X'
+            assert s[1].b == -4892220
+            assert s[1].a == b'Y'
+
+    def test_pack_valueerror(self):
+        ffi = FFI(backend=self.Backend())
+        py.test.raises(ValueError, ffi.cdef, "", pack=3)
+        py.test.raises(ValueError, ffi.cdef, "", packed=2)
+        py.test.raises(ValueError, ffi.cdef, "", packed=True, pack=1)
 
     def test_define_integer_constant(self):
         ffi = FFI(backend=self.Backend())
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
@@ -2243,6 +2243,12 @@
         "'API mode' and non-variadic (i.e. declared inside ffibuilder.cdef()"
         "+ffibuilder.set_source() and not taking a final '...' argument)")
 
+def test_pack_not_supported():
+    ffi = FFI()
+    ffi.cdef("""struct foo { char y; int x; };""", pack=2)
+    py.test.raises(NotImplementedError, verify,
+                   ffi, "test_pack_not_supported", "")
+
 def test_gcc_visibility_hidden():
     if sys.platform == 'win32':
         py.test.skip("test for gcc/clang")

From pypy.commits at gmail.com  Thu Jul 12 02:01:54 2018
From: pypy.commits at gmail.com (wlav)
Date: Wed, 11 Jul 2018 23:01:54 -0700 (PDT)
Subject: [pypy-commit] pypy cppyy-packaging: do not allow instantiation of
 namespaces
Message-ID: <5b46eed2.1c69fb81.9ae43.b150@mx.google.com>

Author: Wim Lavrijsen <WLavrijsen at lbl.gov>
Branch: cppyy-packaging
Changeset: r94850:5b42ae224e5e
Date: 2018-07-11 22:28 -0700
http://bitbucket.org/pypy/pypy/changeset/5b42ae224e5e/

Log:	do not allow instantiation of namespaces

diff --git a/pypy/module/_cppyy/pythonify.py b/pypy/module/_cppyy/pythonify.py
--- a/pypy/module/_cppyy/pythonify.py
+++ b/pypy/module/_cppyy/pythonify.py
@@ -73,7 +73,8 @@
 
 # C++ namespace base class (the C++ class base class defined in _post_import_startup)
 class CPPNamespace(with_metaclass(CPPNamespaceMeta, object)):
-     pass
+    def __init__(self):
+        raise TypeError("cannot instantiate namespace '%s'", self.__cppname__)
 
 
 # TODO: this can be moved to the interp level (and share template argument
diff --git a/pypy/module/_cppyy/test/test_advancedcpp.py b/pypy/module/_cppyy/test/test_advancedcpp.py
--- a/pypy/module/_cppyy/test/test_advancedcpp.py
+++ b/pypy/module/_cppyy/test/test_advancedcpp.py
@@ -148,6 +148,8 @@
         assert gbl.a_ns.d_ns.e_class.f_class.s_f      == 66
         assert gbl.a_ns.d_ns.e_class.f_class().m_f    == -6
 
+        raises(TypeError, gbl.a_ns)
+
     def test03a_namespace_lookup_on_update(self):
         """Test whether namespaces can be shared across dictionaries."""
 

From pypy.commits at gmail.com  Thu Jul 12 02:01:57 2018
From: pypy.commits at gmail.com (wlav)
Date: Wed, 11 Jul 2018 23:01:57 -0700 (PDT)
Subject: [pypy-commit] pypy cppyy-packaging: factor out abstract class
 constructors
Message-ID: <5b46eed5.1c69fb81.b94a5.cb1a@mx.google.com>

Author: Wim Lavrijsen <WLavrijsen at lbl.gov>
Branch: cppyy-packaging
Changeset: r94851:534891668842
Date: 2018-07-11 22:29 -0700
http://bitbucket.org/pypy/pypy/changeset/534891668842/

Log:	factor out abstract class constructors

diff --git a/pypy/module/_cppyy/interp_cppyy.py b/pypy/module/_cppyy/interp_cppyy.py
--- a/pypy/module/_cppyy/interp_cppyy.py
+++ b/pypy/module/_cppyy/interp_cppyy.py
@@ -657,11 +657,6 @@
     @unwrap_spec(args_w='args_w')
     def call_args(self, args_w):
         jit.promote(self)
-        # TODO: factor out the following:
-        if capi.c_is_abstract(self.space, self.scope.handle):
-            raise oefmt(self.space.w_TypeError,
-                        "cannot instantiate abstract class '%s'",
-                        self.scope.name)
         cppinstance = self.space.interp_w(W_CPPInstance, args_w[0])
         w_result = self.call_impl(rffi.cast(capi.C_OBJECT, self.scope.handle), args_w[1:])
         newthis = rffi.cast(capi.C_OBJECT, self.space.uint_w(w_result))
@@ -679,6 +674,23 @@
     __doc__    = GetSetProperty(W_CPPConstructorOverload.fget_doc)
 )
 
+class W_CPPAbstractConstructorOverload(W_CPPOverload):
+    _attrs_ = []
+
+    @unwrap_spec(args_w='args_w')
+    def call_args(self, args_w):
+        raise oefmt(self.space.w_TypeError,
+            "cannot instantiate abstract class '%s'", self.scope.name)
+
+    def __repr__(self):
+        return "W_CPPAbstractConstructorOverload"
+
+W_CPPAbstractConstructorOverload.typedef = TypeDef(
+    'CPPAbstractConstructorOverload',
+    __get__    = interp2app(W_CPPConstructorOverload.descr_get),
+    __call__   = interp2app(W_CPPConstructorOverload.call_args),
+)
+
 
 class TemplateOverloadMixin(object):
     """Mixin to instantiate templated methods/functions."""
@@ -1210,7 +1222,10 @@
             ftype = ftype_tmp[pyname]
             CPPMethodSort(methods).sort()
             if ftype & FUNCTION_IS_CONSTRUCTOR:
-                overload = W_CPPConstructorOverload(self.space, self, methods[:])
+                if capi.c_is_abstract(self.space, self.handle):
+                    overload = W_CPPAbstractConstructorOverload(self.space, self, methods[:])
+                else:
+                    overload = W_CPPConstructorOverload(self.space, self, methods[:])
             elif ftype & FUNCTION_IS_STATIC:
                 if ftype & FUNCTION_IS_TEMPLATE:
                     cppname = capi.c_method_name(self.space, methods[0].cppmethod)

From pypy.commits at gmail.com  Thu Jul 12 02:02:01 2018
From: pypy.commits at gmail.com (wlav)
Date: Wed, 11 Jul 2018 23:02:01 -0700 (PDT)
Subject: [pypy-commit] pypy cppyy-packaging: bring in a load of tests from
 cppyy/test
Message-ID: <5b46eed9.1c69fb81.a4e93.7d9d@mx.google.com>

Author: Wim Lavrijsen <WLavrijsen at lbl.gov>
Branch: cppyy-packaging
Changeset: r94852:8fe8681b8102
Date: 2018-07-11 22:41 -0700
http://bitbucket.org/pypy/pypy/changeset/8fe8681b8102/

Log:	bring in a load of tests from cppyy/test

diff --git a/pypy/module/_cppyy/test/advancedcpp.cxx b/pypy/module/_cppyy/test/advancedcpp.cxx
--- a/pypy/module/_cppyy/test/advancedcpp.cxx
+++ b/pypy/module/_cppyy/test/advancedcpp.cxx
@@ -4,20 +4,32 @@
 
 
 // for testing of default arguments
-#define IMPLEMENT_DEFAULTER_CLASS(type, tname)                               \
+#define IMPLEMENT_DEFAULTERS(type, tname)                                    \
 tname##_defaulter::tname##_defaulter(type a, type b, type c) {               \
-   m_a = a; m_b = b; m_c = c;                                                \
+    m_a = a; m_b = b; m_c = c;                                               \
+}                                                                            \
+type tname##_defaulter_func(int idx, type a, type b, type c) {               \
+    if (idx == 0) return a;                                                  \
+    if (idx == 1) return b;                                                  \
+    if (idx == 2) return c;                                                  \
+    return (type)idx;                                                        \
 }
-IMPLEMENT_DEFAULTER_CLASS(short, short)
-IMPLEMENT_DEFAULTER_CLASS(unsigned short, ushort)
-IMPLEMENT_DEFAULTER_CLASS(int, int)
-IMPLEMENT_DEFAULTER_CLASS(unsigned, uint)
-IMPLEMENT_DEFAULTER_CLASS(long, long)
-IMPLEMENT_DEFAULTER_CLASS(unsigned long, ulong)
-IMPLEMENT_DEFAULTER_CLASS(long long, llong)
-IMPLEMENT_DEFAULTER_CLASS(unsigned long long, ullong)
-IMPLEMENT_DEFAULTER_CLASS(float, float)
-IMPLEMENT_DEFAULTER_CLASS(double, double)
+IMPLEMENT_DEFAULTERS(short, short)
+IMPLEMENT_DEFAULTERS(unsigned short, ushort)
+IMPLEMENT_DEFAULTERS(int, int)
+IMPLEMENT_DEFAULTERS(unsigned, uint)
+IMPLEMENT_DEFAULTERS(long, long)
+IMPLEMENT_DEFAULTERS(unsigned long, ulong)
+IMPLEMENT_DEFAULTERS(long long, llong)
+IMPLEMENT_DEFAULTERS(unsigned long long, ullong)
+IMPLEMENT_DEFAULTERS(float, float)
+IMPLEMENT_DEFAULTERS(double, double)
+
+std::string string_defaulter_func(int idx, const std::string& name1, std::string name2) {
+    if (idx == 0) return name1;
+    if (idx == 1) return name2;
+    return "mies";
+}
 
 
 // for esoteric inheritance testing
@@ -77,11 +89,11 @@
 double my_global_array[500];
 static double sd = 1234.;
 double* my_global_ptr = &sd;
+const char my_global_string2[] = "zus jet teun";
 some_int_holder my_global_int_holders[5] = {
     some_int_holder(13), some_int_holder(42), some_int_holder(88),
     some_int_holder(-1), some_int_holder(17) };
 
-
 // for life-line and identity testing
 int some_class_with_data::some_data::s_num_data = 0;
 
diff --git a/pypy/module/_cppyy/test/advancedcpp.h b/pypy/module/_cppyy/test/advancedcpp.h
--- a/pypy/module/_cppyy/test/advancedcpp.h
+++ b/pypy/module/_cppyy/test/advancedcpp.h
@@ -4,24 +4,27 @@
 
 
 //===========================================================================
-#define DECLARE_DEFAULTER_CLASS(type, tname)                                \
+#define DECLARE_DEFAULTERS(type, tname)                                     \
 class tname##_defaulter {                                                   \
 public:                                                                     \
     tname##_defaulter(type a = 11, type b = 22, type c = 33);               \
                                                                             \
 public:                                                                     \
     type m_a, m_b, m_c;                                                     \
-};
-DECLARE_DEFAULTER_CLASS(short, short)   // for testing of default arguments
-DECLARE_DEFAULTER_CLASS(unsigned short, ushort)
-DECLARE_DEFAULTER_CLASS(int, int)
-DECLARE_DEFAULTER_CLASS(unsigned, uint)
-DECLARE_DEFAULTER_CLASS(long, long)
-DECLARE_DEFAULTER_CLASS(unsigned long, ulong)
-DECLARE_DEFAULTER_CLASS(long long, llong)
-DECLARE_DEFAULTER_CLASS(unsigned long long, ullong)
-DECLARE_DEFAULTER_CLASS(float, float)
-DECLARE_DEFAULTER_CLASS(double, double)
+};                                                                          \
+type tname##_defaulter_func(int idx = 0, type a = 11, type b = 22, type c = 33);
+DECLARE_DEFAULTERS(short, short)   // for testing of default arguments
+DECLARE_DEFAULTERS(unsigned short, ushort)
+DECLARE_DEFAULTERS(int, int)
+DECLARE_DEFAULTERS(unsigned, uint)
+DECLARE_DEFAULTERS(long, long)
+DECLARE_DEFAULTERS(unsigned long, ulong)
+DECLARE_DEFAULTERS(long long, llong)
+DECLARE_DEFAULTERS(unsigned long long, ullong)
+DECLARE_DEFAULTERS(float, float)
+DECLARE_DEFAULTERS(double, double)
+
+std::string string_defaulter_func(int idx, const std::string& name1 = "aap", std::string name2 = "noot");
 
 
 //===========================================================================
@@ -274,7 +277,8 @@
 extern double my_global_double;    // a couple of globals for access testing
 extern double my_global_array[500];
 extern double* my_global_ptr;
-static const char my_global_string[] = "aap " " noot " " mies";
+static const char my_global_string1[] = "aap " " noot " " mies";
+extern const char my_global_string2[];
 
 class some_int_holder {
 public:
@@ -311,6 +315,11 @@
     some_data m_data;
 };
 
+class refers_to_self {             // for data member reuse testing
+public:
+    refers_to_self* m_other = nullptr;
+};
+
 
 //===========================================================================
 class pointer_pass {               // for testing passing of void*'s
@@ -419,3 +428,34 @@
     void throw_anything();
     void throw_exception();
 };
+
+
+//===========================================================================
+class UsingBase {                  // using declaration testing
+public:
+    UsingBase(int n = 13) : m_int(n) {} 
+    virtual char vcheck() { return 'A'; }
+    int m_int;
+};
+
+class UsingDerived : public UsingBase {
+public:
+    using UsingBase::UsingBase;
+    virtual char vcheck() { return 'B'; }
+    int m_int2 = 42;
+};
+
+
+//===========================================================================
+class TypedefToPrivateClass {      // typedef resolution testing
+private:
+    class PC {
+    public:
+        PC(int i) : m_val(i) {}
+        int m_val;
+    };
+
+public:
+    typedef PC PP;
+    PP f() { return PC(42); }
+};
diff --git a/pypy/module/_cppyy/test/advancedcpp.xml b/pypy/module/_cppyy/test/advancedcpp.xml
--- a/pypy/module/_cppyy/test/advancedcpp.xml
+++ b/pypy/module/_cppyy/test/advancedcpp.xml
@@ -1,6 +1,7 @@
 <lcgdict>
 
   <class pattern="*_defaulter" />
+  <function pattern="*_defaulter_func" />
 
   <class name="base_class" />
   <class name="derived_class" />
@@ -32,6 +33,7 @@
   <class name="some_convertible" />
   <class name="some_class_with_data" />
   <class name="some_class_with_data::some_data" />
+  <class name="refers_to_self" />
 
   <class name="some_comparable" />
   <function name="operator==" />
@@ -59,5 +61,7 @@
   <class name="overload_the_other_way" />
 
   <class name="Thrower" />
+  <class pattern="Using*" />
+  <class name="TypedefToPrivateClass" />
 
 </lcgdict>
diff --git a/pypy/module/_cppyy/test/test_advancedcpp.py b/pypy/module/_cppyy/test/test_advancedcpp.py
--- a/pypy/module/_cppyy/test/test_advancedcpp.py
+++ b/pypy/module/_cppyy/test/test_advancedcpp.py
@@ -56,6 +56,12 @@
             assert d.m_b ==  t(4)
             assert d.m_c ==  t(5)
             d.__destruct__()
+
+            defaulter_func = getattr(cppyy.gbl, '%s_defaulter_func' %n)
+            answers = [11, 22, 33, 3]
+            for idx in range(4):
+                assert defaulter_func(idx) == answers[idx]
+
         test_defaulter('short',  int)
         test_defaulter('ushort', int)
         test_defaulter('int',    int)
@@ -67,6 +73,13 @@
         test_defaulter('float',  float)
         test_defaulter('double', float)
 
+        assert cppyy.gbl.string_defaulter_func(0)               == "aap"
+        assert cppyy.gbl.string_defaulter_func(0, "zus")        == "zus"
+        assert cppyy.gbl.string_defaulter_func(1)               == "noot"
+        assert cppyy.gbl.string_defaulter_func(1, "zus")        == "noot"
+        assert cppyy.gbl.string_defaulter_func(1, "zus", "jet") == "jet"
+        assert cppyy.gbl.string_defaulter_func(2)               == "mies"
+
     def test02_simple_inheritance(self):
         """Test binding of a basic inheritance structure"""
 
@@ -655,10 +668,18 @@
 
         assert cppyy.gbl.my_global_double == 12.
         assert len(cppyy.gbl.my_global_array) == 500
-        assert cppyy.gbl.my_global_string == "aap  noot  mies"
+        assert cppyy.gbl.my_global_string1 == "aap  noot  mies"
+        return     # next line currently crashes
+        assert cppyy.gbl.my_global_string2 == "zus jet teun"
         # TODO: currently fails b/c double** not understood as &double*
         #assert cppyy.gbl.my_global_ptr[0] == 1234.
 
+        v = cppyy.gbl.my_global_int_holders
+        assert len(v) == 5
+        expected_vals = [13, 42, 88, -1, 17]
+        for i in range(len(v)):
+            assert v[i].m_val == expected_vals[i]
+
     def test22_exceptions(self):
         """Catching of C++ exceptions"""
 
@@ -677,3 +698,35 @@
             t.throw_exception()
         except Exception as e:
             "C++ function failed" in str(e)
+
+    def test23_using(self):
+        """Accessibility of using declarations"""
+
+        import _cppyy as cppyy
+
+        assert cppyy.gbl.UsingBase().vcheck() == 'A'
+
+        B = cppyy.gbl.UsingDerived
+        assert not 'UsingBase' in B.__init__.__doc__
+
+        b1 = B()
+        assert b1.m_int    == 13
+        assert b1.m_int2   == 42
+        assert b1.vcheck() == 'B'
+ 
+        b2 = B(10)
+        assert b2.m_int    == 10
+        assert b2.m_int2   == 42
+        assert b2.vcheck() == 'B'
+
+        b3 = B(b2)
+        assert b3.m_int    == 10
+        assert b3.m_int2   == 42
+        assert b3.vcheck() == 'B'
+
+    def test24_typedef_to_private_class(self):
+        """Typedefs to private classes should not resolve"""
+
+        import _cppyy as cppyy
+
+        assert cppyy.gbl.TypedefToPrivateClass().f().m_val == 42

From pypy.commits at gmail.com  Fri Jul 13 02:16:45 2018
From: pypy.commits at gmail.com (wlav)
Date: Thu, 12 Jul 2018 23:16:45 -0700 (PDT)
Subject: [pypy-commit] pypy cppyy-packaging: fix typos
Message-ID: <5b4843cd.1c69fb81.b94ae.7d52@mx.google.com>

Author: Wim Lavrijsen <WLavrijsen at lbl.gov>
Branch: cppyy-packaging
Changeset: r94853:63e440d921c8
Date: 2018-07-12 07:46 -0700
http://bitbucket.org/pypy/pypy/changeset/63e440d921c8/

Log:	fix typos

diff --git a/pypy/module/_cppyy/interp_cppyy.py b/pypy/module/_cppyy/interp_cppyy.py
--- a/pypy/module/_cppyy/interp_cppyy.py
+++ b/pypy/module/_cppyy/interp_cppyy.py
@@ -687,8 +687,8 @@
 
 W_CPPAbstractConstructorOverload.typedef = TypeDef(
     'CPPAbstractConstructorOverload',
-    __get__    = interp2app(W_CPPConstructorOverload.descr_get),
-    __call__   = interp2app(W_CPPConstructorOverload.call_args),
+    __get__    = interp2app(W_CPPAbstractConstructorOverload.descr_get),
+    __call__   = interp2app(W_CPPAbstractConstructorOverload.call_args),
 )
 
 

From pypy.commits at gmail.com  Fri Jul 13 02:16:47 2018
From: pypy.commits at gmail.com (wlav)
Date: Thu, 12 Jul 2018 23:16:47 -0700 (PDT)
Subject: [pypy-commit] pypy cppyy-packaging: improve object identity
 handling (now also includes data members)
Message-ID: <5b4843cf.1c69fb81.6adbb.ec30@mx.google.com>

Author: Wim Lavrijsen <WLavrijsen at lbl.gov>
Branch: cppyy-packaging
Changeset: r94854:80195e50eca4
Date: 2018-07-12 12:39 -0700
http://bitbucket.org/pypy/pypy/changeset/80195e50eca4/

Log:	improve object identity handling (now also includes data members)

diff --git a/pypy/module/_cppyy/converter.py b/pypy/module/_cppyy/converter.py
--- a/pypy/module/_cppyy/converter.py
+++ b/pypy/module/_cppyy/converter.py
@@ -566,10 +566,6 @@
         from pypy.module._cppyy import interp_cppyy
         return interp_cppyy.wrap_cppinstance(space, address, self.clsdecl, do_cast=False)
 
-    def to_memory(self, space, w_obj, w_value, offset):
-        address = rffi.cast(rffi.VOIDPP, self._get_raw_address(space, w_obj, offset))
-        address[0] = rffi.cast(rffi.VOIDP, self._unwrap_object(space, w_value))
-
 class InstancePtrPtrConverter(InstancePtrConverter):
     typecode = 'o'
 
@@ -597,6 +593,25 @@
         return interp_cppyy.wrap_cppinstance(
             space, address, self.clsdecl, do_cast=False, is_ref=True)
 
+    def to_memory(self, space, w_obj, w_value, offset):
+        # the actual data member is of object* type, but we receive a pointer to that
+        # data member in order to modify its value, so by convention, the internal type
+        # used is object**
+        address = rffi.cast(rffi.VOIDPP, self._get_raw_address(space, w_obj, offset))
+        from pypy.module._cppyy.interp_cppyy import W_CPPInstance
+        cppinstance = space.interp_w(W_CPPInstance, w_value, can_be_None=True)
+        if cppinstance:
+            rawobject = cppinstance.get_rawobject()
+            offset = capi.c_base_offset(space, cppinstance.clsdecl, self.clsdecl, rawobject, 1)
+            obj_address = capi.direct_ptradd(rawobject, offset)
+            address[0] = rffi.cast(rffi.VOIDP, obj_address);
+            # register the value for potential recycling
+            from pypy.module._cppyy.interp_cppyy import memory_regulator
+            memory_regulator.register(cppinstance)
+        else:
+            raise oefmt(space.w_TypeError,
+                "cannot pass %T instance as %s", w_value, self.clsdecl.name)
+
     def finalize_call(self, space, w_obj):
         if self.ref_buffer:
             set_rawobject(space, w_obj, self.ref_buffer[0])
diff --git a/pypy/module/_cppyy/interp_cppyy.py b/pypy/module/_cppyy/interp_cppyy.py
--- a/pypy/module/_cppyy/interp_cppyy.py
+++ b/pypy/module/_cppyy/interp_cppyy.py
@@ -167,6 +167,7 @@
 #
 #  W_CPPOverload:                 instance methods (base class)
 #  W_CPPConstructorOverload:      constructors
+#  W_CPPAbstractCtorOverload:     to provent instantiation of abstract classes
 #  W_CPPStaticOverload:           free and static functions
 #  W_CPPTemplateOverload:         templated methods
 #  W_CPPTemplateStaticOverload:   templated free and static functions
@@ -674,7 +675,7 @@
     __doc__    = GetSetProperty(W_CPPConstructorOverload.fget_doc)
 )
 
-class W_CPPAbstractConstructorOverload(W_CPPOverload):
+class W_CPPAbstractCtorOverload(W_CPPOverload):
     _attrs_ = []
 
     @unwrap_spec(args_w='args_w')
@@ -683,12 +684,12 @@
             "cannot instantiate abstract class '%s'", self.scope.name)
 
     def __repr__(self):
-        return "W_CPPAbstractConstructorOverload"
+        return "W_CPPAbstractCtorOverload"
 
-W_CPPAbstractConstructorOverload.typedef = TypeDef(
-    'CPPAbstractConstructorOverload',
-    __get__    = interp2app(W_CPPAbstractConstructorOverload.descr_get),
-    __call__   = interp2app(W_CPPAbstractConstructorOverload.call_args),
+W_CPPAbstractCtorOverload.typedef = TypeDef(
+    'CPPAbstractCtorOverload',
+    __get__    = interp2app(W_CPPAbstractCtorOverload.descr_get),
+    __call__   = interp2app(W_CPPAbstractCtorOverload.call_args),
 )
 
 
@@ -1086,7 +1087,7 @@
         sig = '(%s)' % signature
         for f in overload.functions:
             if f.signature(False) == sig:
-                return W_CPPOverload(self.space, self, [f])
+                return type(overload)(self.space, self, [f])
         raise oefmt(self.space.w_LookupError, "no overload matches signature")
 
     def __eq__(self, other):
@@ -1181,9 +1182,13 @@
 
 
 class W_CPPClassDecl(W_CPPScopeDecl):
-    _attrs_ = ['space', 'handle', 'name', 'overloads', 'datamembers']
+    _attrs_ = ['space', 'handle', 'name', 'overloads', 'datamembers', 'cppobjects']
     _immutable_fields_ = ['handle', 'name', 'overloads[*]', 'datamembers[*]']
 
+    def __init__(self, space, opaque_handle, final_scoped_name):
+        W_CPPScopeDecl.__init__(self, space, opaque_handle, final_scoped_name)
+        self.cppobjects = rweakref.RWeakValueDictionary(int, W_CPPInstance)
+
     def _build_overloads(self):
         assert len(self.overloads) == 0
         methods_tmp = {}; ftype_tmp = {}
@@ -1223,7 +1228,7 @@
             CPPMethodSort(methods).sort()
             if ftype & FUNCTION_IS_CONSTRUCTOR:
                 if capi.c_is_abstract(self.space, self.handle):
-                    overload = W_CPPAbstractConstructorOverload(self.space, self, methods[:])
+                    overload = W_CPPAbstractCtorOverload(self.space, self, methods[:])
                 else:
                     overload = W_CPPConstructorOverload(self.space, self, methods[:])
             elif ftype & FUNCTION_IS_STATIC:
@@ -1543,31 +1548,33 @@
 
 
 class MemoryRegulator:
-    # TODO: (?) An object address is not unique if e.g. the class has a
-    # public data member of class type at the start of its definition and
-    # has no virtual functions. A _key class that hashes on address and
-    # type would be better, but my attempt failed in the rtyper, claiming
-    # a call on None ("None()") and needed a default ctor. (??)
-    # Note that for now, the associated test carries an m_padding to make
-    # a difference in the addresses.
-    def __init__(self):
-        self.objects = rweakref.RWeakValueDictionary(int, W_CPPInstance)
+    _immutable_ = True
 
-    def register(self, obj):
+    @staticmethod
+    def register(obj):
         if not obj._rawobject:
             return
-        int_address = int(rffi.cast(rffi.LONG, obj._rawobject))
-        self.objects.set(int_address, obj)
+        addr_as_int = int(rffi.cast(rffi.LONG, obj.get_rawobject()))
+        clsdecl = obj.clsdecl
+        assert isinstance(clsdecl, W_CPPClassDecl)
+        clsdecl.cppobjects.set(addr_as_int, obj)
 
-    def unregister(self, obj):
+    @staticmethod
+    def unregister(obj):
         if not obj._rawobject:
             return
-        int_address = int(rffi.cast(rffi.LONG, obj._rawobject))
-        self.objects.set(int_address, None)
+        addr_as_int = int(rffi.cast(rffi.LONG, obj.get_rawobject()))
+        clsdecl = obj.clsdecl
+        assert isinstance(clsdecl, W_CPPClassDecl)
+        clsdecl.cppobjects.set(addr_as_int, None) # actually deletes (pops)
 
-    def retrieve(self, address):
-        int_address = int(rffi.cast(rffi.LONG, address))
-        return self.objects.get(int_address)
+    @staticmethod
+    def retrieve(clsdecl, address):
+        if not address:
+            return None
+        addr_as_int = int(rffi.cast(rffi.LONG, address))
+        assert isinstance(clsdecl, W_CPPClassDecl)
+        return clsdecl.cppobjects.get(addr_as_int)
 
 memory_regulator = MemoryRegulator()
 
@@ -1613,8 +1620,11 @@
 
     # try to recycle existing object if this one is not newly created
     if not fresh and rawobject:
-        obj = memory_regulator.retrieve(rawobject)
-        if obj is not None and obj.clsdecl is clsdecl:
+        address = rawobject
+        if is_ref:
+            address = rffi.cast(capi.C_OBJECT, rffi.cast(rffi.VOIDPP, address)[0])
+        obj = memory_regulator.retrieve(clsdecl, address)
+        if obj is not None:
             return obj
 
     # fresh creation
diff --git a/pypy/module/_cppyy/test/test_advancedcpp.py b/pypy/module/_cppyy/test/test_advancedcpp.py
--- a/pypy/module/_cppyy/test/test_advancedcpp.py
+++ b/pypy/module/_cppyy/test/test_advancedcpp.py
@@ -489,6 +489,23 @@
         d2.__destruct__()
         d1.__destruct__()
 
+        RTS = cppyy.gbl.refers_to_self
+
+        r1 = RTS()
+        r2 = RTS()
+        r1.m_other = r2
+
+        r3 = r1.m_other
+        r4 = r1.m_other
+        assert r3 is r4
+
+        assert r3 == r2
+        assert r3 is r2
+
+        r3.extra = 42
+        assert r2.extra == 42
+        assert r4.extra == 42
+
     def test11_multi_methods(self):
         """Test calling of methods from multiple inheritance"""
 

From pypy.commits at gmail.com  Fri Jul 13 02:16:49 2018
From: pypy.commits at gmail.com (wlav)
Date: Thu, 12 Jul 2018 23:16:49 -0700 (PDT)
Subject: [pypy-commit] pypy cppyy-packaging: do not unwrap assignable in
 setitem until call with ref-return is done
Message-ID: <5b4843d1.1c69fb81.3c461.5345@mx.google.com>

Author: Wim Lavrijsen <WLavrijsen at lbl.gov>
Branch: cppyy-packaging
Changeset: r94855:0097f52d422a
Date: 2018-07-12 19:54 -0700
http://bitbucket.org/pypy/pypy/changeset/0097f52d422a/

Log:	do not unwrap assignable in setitem until call with ref-return is
	done

diff --git a/pypy/module/_cppyy/executor.py b/pypy/module/_cppyy/executor.py
--- a/pypy/module/_cppyy/executor.py
+++ b/pypy/module/_cppyy/executor.py
@@ -98,10 +98,10 @@
     def __init__(self, space, extra):
         Executor.__init__(self, space, extra)
         self.do_assign = False
-        self.item = rffi.cast(self.c_type, 0)
+        self.w_item = space.w_None
 
     def set_item(self, space, w_item):
-        self.item = self._unwrap_object(space, w_item)
+        self.w_item = w_item
         self.do_assign = True
 
     #def _wrap_object(self, space, obj):
@@ -109,7 +109,7 @@
 
     def _wrap_reference(self, space, rffiptr):
         if self.do_assign:
-            rffiptr[0] = self.item
+            rffiptr[0] = self._unwrap_object(space, self.w_item)
         self.do_assign = False
         return self._wrap_object(space, rffiptr[0])    # all paths, for rtyper
 

From pypy.commits at gmail.com  Fri Jul 13 02:16:51 2018
From: pypy.commits at gmail.com (wlav)
Date: Thu, 12 Jul 2018 23:16:51 -0700 (PDT)
Subject: [pypy-commit] pypy cppyy-packaging: moves for strings (incl. from
 temporary python str)
Message-ID: <5b4843d3.1c69fb81.fc27f.687c@mx.google.com>

Author: Wim Lavrijsen <WLavrijsen at lbl.gov>
Branch: cppyy-packaging
Changeset: r94856:8820afbc98c3
Date: 2018-07-12 20:23 -0700
http://bitbucket.org/pypy/pypy/changeset/8820afbc98c3/

Log:	moves for strings (incl. from temporary python str)

diff --git a/pypy/module/_cppyy/converter.py b/pypy/module/_cppyy/converter.py
--- a/pypy/module/_cppyy/converter.py
+++ b/pypy/module/_cppyy/converter.py
@@ -622,7 +622,6 @@
             self.ref_buffer = lltype.nullptr(rffi.VOIDPP.TO)
 
 class StdStringConverter(InstanceConverter):
-
     def __init__(self, space, extra):
         from pypy.module._cppyy import interp_cppyy
         cppclass = interp_cppyy.scope_byname(space, capi.std_string_name)
@@ -648,6 +647,34 @@
     def free_argument(self, space, arg):
         capi.c_destruct(space, self.clsdecl, rffi.cast(capi.C_OBJECT, rffi.cast(rffi.VOIDPP, arg)[0]))
 
+class StdStringMoveConverter(StdStringConverter):
+    def _unwrap_object(self, space, w_obj):
+        # moving is same as by-ref, but have to check that move is allowed
+        moveit_reason = 3
+        from pypy.module._cppyy.interp_cppyy import W_CPPInstance, INSTANCE_FLAGS_IS_RVALUE
+        try:
+            obj = space.interp_w(W_CPPInstance, w_obj)
+            if obj and obj.rt_flags & INSTANCE_FLAGS_IS_RVALUE:
+                obj.rt_flags &= ~INSTANCE_FLAGS_IS_RVALUE
+                moveit_reason = 1
+            else:
+                moveit_reason = 0
+        except:
+            pass
+
+        if moveit_reason:
+            try:
+                return StdStringConverter._unwrap_object(self, space, w_obj)
+            except Exception:
+                 if moveit_reason == 1:
+                    # TODO: if the method fails on some other converter, then the next
+                    # overload can not be an rvalue anymore
+                    obj = space.interp_w(W_CPPInstance, w_obj)
+                    obj.rt_flags |= INSTANCE_FLAGS_IS_RVALUE
+                    raise
+
+        raise oefmt(space.w_ValueError, "object is not an rvalue")
+
 class StdStringRefConverter(InstancePtrConverter):
     _immutable_fields_ = ['cppclass', 'typecode']
     typecode    = 'V'
@@ -900,6 +927,7 @@
 _converters["std::basic_string<char>"]           = StdStringConverter
 _converters["const std::basic_string<char>&"]    = StdStringConverter     # TODO: shouldn't copy
 _converters["std::basic_string<char>&"]          = StdStringRefConverter
+_converters["std::basic_string<char>&&"]         = StdStringMoveConverter
 
 _converters["PyObject*"]                         = PyObjectConverter
 
@@ -1017,6 +1045,7 @@
         ("std::basic_string<char>",         "string"),
         ("const std::basic_string<char>&",  "const string&"),
         ("std::basic_string<char>&",        "string&"),
+        ("std::basic_string<char>&&",       "string&&"),
 
         ("PyObject*",                       "_object*"),
     )
diff --git a/pypy/module/_cppyy/test/test_stltypes.py b/pypy/module/_cppyy/test/test_stltypes.py
--- a/pypy/module/_cppyy/test/test_stltypes.py
+++ b/pypy/module/_cppyy/test/test_stltypes.py
@@ -22,12 +22,12 @@
     def test01_builtin_type_vector_types(self):
         """Test access to std::vector<int>/std::vector<double>"""
 
-        import _cppyy
+        import _cppyy as cppyy
 
-        assert _cppyy.gbl.std        is _cppyy.gbl.std
-        assert _cppyy.gbl.std.vector is _cppyy.gbl.std.vector
+        assert cppyy.gbl.std        is cppyy.gbl.std
+        assert cppyy.gbl.std.vector is cppyy.gbl.std.vector
 
-        assert callable(_cppyy.gbl.std.vector)
+        assert callable(cppyy.gbl.std.vector)
 
         type_info = (
             ("int",     int),
@@ -36,10 +36,10 @@
         )
 
         for c_type, p_type in type_info:
-            tv1 = getattr(_cppyy.gbl.std, 'vector<%s>' % c_type)
-            tv2 = _cppyy.gbl.std.vector(p_type)
+            tv1 = getattr(cppyy.gbl.std, 'vector<%s>' % c_type)
+            tv2 = cppyy.gbl.std.vector(p_type)
             assert tv1 is tv2
-            assert tv1.iterator is _cppyy.gbl.std.vector(p_type).iterator
+            assert tv1.iterator is cppyy.gbl.std.vector(p_type).iterator
 
             #----- 
             v = tv1(); v += range(self.N)    # default args from Reflex are useless :/
@@ -73,16 +73,16 @@
     def test02_user_type_vector_type(self):
         """Test access to an std::vector<just_a_class>"""
 
-        import _cppyy
+        import _cppyy as cppyy
 
-        assert _cppyy.gbl.std        is _cppyy.gbl.std
-        assert _cppyy.gbl.std.vector is _cppyy.gbl.std.vector
+        assert cppyy.gbl.std        is cppyy.gbl.std
+        assert cppyy.gbl.std.vector is cppyy.gbl.std.vector
 
-        assert callable(_cppyy.gbl.std.vector)
+        assert callable(cppyy.gbl.std.vector)
 
-        tv1 = getattr(_cppyy.gbl.std, 'vector<just_a_class>')
-        tv2 = _cppyy.gbl.std.vector('just_a_class')
-        tv3 = _cppyy.gbl.std.vector(_cppyy.gbl.just_a_class)
+        tv1 = getattr(cppyy.gbl.std, 'vector<just_a_class>')
+        tv2 = cppyy.gbl.std.vector('just_a_class')
+        tv3 = cppyy.gbl.std.vector(cppyy.gbl.just_a_class)
 
         assert tv1 is tv2
         assert tv2 is tv3
@@ -95,7 +95,7 @@
         assert hasattr(v, 'end' )
 
         for i in range(self.N):
-            v.push_back(_cppyy.gbl.just_a_class())
+            v.push_back(cppyy.gbl.just_a_class())
             v[i].m_i = i
             assert v[i].m_i == i
 
@@ -105,9 +105,9 @@
     def test03_empty_vector_type(self):
         """Test behavior of empty std::vector<int>"""
 
-        import _cppyy
+        import _cppyy as cppyy
 
-        v = _cppyy.gbl.std.vector(int)()
+        v = cppyy.gbl.std.vector(int)()
         for arg in v:
             pass
         v.__destruct__()
@@ -115,9 +115,9 @@
     def test04_vector_iteration(self):
         """Test iteration over an std::vector<int>"""
 
-        import _cppyy
+        import _cppyy as cppyy
 
-        v = _cppyy.gbl.std.vector(int)()
+        v = cppyy.gbl.std.vector(int)()
 
         for i in range(self.N):
             v.push_back(i)
@@ -140,9 +140,9 @@
     def test05_push_back_iterables_with_iadd(self):
         """Test usage of += of iterable on push_back-able container"""
 
-        import _cppyy
+        import _cppyy as cppyy
 
-        v = _cppyy.gbl.std.vector(int)()
+        v = cppyy.gbl.std.vector(int)()
 
         v += [1, 2, 3]
         assert len(v) == 3
@@ -159,7 +159,7 @@
         raises(TypeError, v.__iadd__, (7, '8'))  # string shouldn't pass
         assert len(v) == 7   # TODO: decide whether this should roll-back
 
-        v2 = _cppyy.gbl.std.vector(int)()
+        v2 = cppyy.gbl.std.vector(int)()
         v2 += [8, 9]
         assert len(v2) == 2
         assert v2[0] == 8
@@ -174,9 +174,9 @@
     def test06_vector_indexing(self):
         """Test python-style indexing to an std::vector<int>"""
 
-        import _cppyy
+        import _cppyy as cppyy
 
-        v = _cppyy.gbl.std.vector(int)()
+        v = cppyy.gbl.std.vector(int)()
 
         for i in range(self.N):
             v.push_back(i)
@@ -209,9 +209,9 @@
     def test01_string_argument_passing(self):
         """Test mapping of python strings and std::string"""
 
-        import _cppyy
-        std = _cppyy.gbl.std
-        stringy_class = _cppyy.gbl.stringy_class
+        import _cppyy as cppyy
+        std = cppyy.gbl.std
+        stringy_class = cppyy.gbl.stringy_class
 
         c, s = stringy_class(""), std.string("test1")
 
@@ -240,9 +240,9 @@
     def test02_string_data_access(self):
         """Test access to std::string object data members"""
 
-        import _cppyy
-        std = _cppyy.gbl.std
-        stringy_class = _cppyy.gbl.stringy_class
+        import _cppyy as cppyy
+        std = cppyy.gbl.std
+        stringy_class = cppyy.gbl.stringy_class
 
         c, s = stringy_class("dummy"), std.string("test string")
 
@@ -261,9 +261,9 @@
 
         return # don't bother; is fixed in cling-support
 
-        import _cppyy
-        std = _cppyy.gbl.std
-        stringy_class = _cppyy.gbl.stringy_class
+        import _cppyy as cppyy
+        std = cppyy.gbl.std
+        stringy_class = cppyy.gbl.stringy_class
 
         t0 = "aap\0noot"
         assert t0 == "aap\0noot"
@@ -288,8 +288,8 @@
     def test01_builtin_list_type(self):
         """Test access to a list<int>"""
 
-        import _cppyy
-        std = _cppyy.gbl.std
+        import _cppyy as cppyy
+        std = cppyy.gbl.std
 
         type_info = (
             ("int",     int),
@@ -299,9 +299,9 @@
 
         for c_type, p_type in type_info:
             tl1 = getattr(std, 'list<%s>' % c_type)
-            tl2 = _cppyy.gbl.std.list(p_type)
+            tl2 = cppyy.gbl.std.list(p_type)
             assert tl1 is tl2
-            assert tl1.iterator is _cppyy.gbl.std.list(p_type).iterator
+            assert tl1.iterator is cppyy.gbl.std.list(p_type).iterator
 
             #-----
             a = tl1()
@@ -323,8 +323,8 @@
     def test02_empty_list_type(self):
         """Test behavior of empty list<int>"""
 
-        import _cppyy
-        std = _cppyy.gbl.std
+        import _cppyy as cppyy
+        std = cppyy.gbl.std
 
         a = std.list(int)()
         for arg in a:
@@ -344,8 +344,8 @@
     def test01_builtin_map_type(self):
         """Test access to a map<int,int>"""
 
-        import _cppyy
-        std = _cppyy.gbl.std
+        import _cppyy as cppyy
+        std = cppyy.gbl.std
 
         a = std.map(int, int)()
         for i in range(self.N):
@@ -373,8 +373,8 @@
     def test02_keyed_maptype(self):
         """Test access to a map<std::string,int>"""
 
-        import _cppyy
-        std = _cppyy.gbl.std
+        import _cppyy as cppyy
+        std = cppyy.gbl.std
 
         a = std.map(std.string, int)()
         for i in range(self.N):
@@ -386,8 +386,8 @@
     def test03_empty_maptype(self):
         """Test behavior of empty map<int,int>"""
 
-        import _cppyy
-        std = _cppyy.gbl.std
+        import _cppyy as cppyy
+        std = cppyy.gbl.std
 
         m = std.map(int, int)()
         for key, value in m:
@@ -396,8 +396,9 @@
     def test04_unsignedvalue_typemap_types(self):
         """Test assignability of maps with unsigned value types"""
 
-        import _cppyy, math, sys
-        std = _cppyy.gbl.std
+        import _cppyy as cppyy
+        import math, sys
+        std = cppyy.gbl.std
 
         mui = std.map(str, 'unsigned int')()
         mui['one'] = 1
@@ -420,8 +421,8 @@
     def test05_STL_like_class_indexing_overloads(self):
         """Test overloading of operator[] in STL like class"""
 
-        import _cppyy
-        stl_like_class = _cppyy.gbl.stl_like_class
+        import _cppyy as cppyy
+        stl_like_class = cppyy.gbl.stl_like_class
 
         a = stl_like_class(int)()
         assert a["some string" ] == 'string'
@@ -430,8 +431,8 @@
     def test06_STL_like_class_iterators(self):
         """Test the iterator protocol mapping for an STL like class"""
 
-        import _cppyy
-        stl_like_class = _cppyy.gbl.stl_like_class
+        import _cppyy as cppyy
+        stl_like_class = cppyy.gbl.stl_like_class
 
         a = stl_like_class(int)()
         for i in a:
@@ -452,8 +453,8 @@
     def test01_builtin_vector_iterators(self):
         """Test iterator comparison with operator== reflected"""
 
-        import _cppyy
-        std = _cppyy.gbl.std
+        import _cppyy as cppyy
+        std = cppyy.gbl.std
 
         v = std.vector(int)()
         v.resize(1)
@@ -489,9 +490,9 @@
     def test01_explicit_templates(self):
         """Explicit use of Template class"""
 
-        import _cppyy
+        import _cppyy as cppyy
 
-        vector = _cppyy.Template('vector', _cppyy.gbl.std)
+        vector = cppyy.Template('vector', cppyy.gbl.std)
         assert vector[int] == vector(int)
 
         v = vector[int]()

From pypy.commits at gmail.com  Fri Jul 13 02:16:53 2018
From: pypy.commits at gmail.com (wlav)
Date: Thu, 12 Jul 2018 23:16:53 -0700 (PDT)
Subject: [pypy-commit] pypy cppyy-packaging: improved overload selection
Message-ID: <5b4843d5.1c69fb81.3706e.3d2d@mx.google.com>

Author: Wim Lavrijsen <WLavrijsen at lbl.gov>
Branch: cppyy-packaging
Changeset: r94857:05fdf73d5e42
Date: 2018-07-12 20:47 -0700
http://bitbucket.org/pypy/pypy/changeset/05fdf73d5e42/

Log:	improved overload selection

diff --git a/pypy/module/_cppyy/interp_cppyy.py b/pypy/module/_cppyy/interp_cppyy.py
--- a/pypy/module/_cppyy/interp_cppyy.py
+++ b/pypy/module/_cppyy/interp_cppyy.py
@@ -41,6 +41,7 @@
              'void**'        : 100,
              'float'         :  30,
              'double'        :  10,
+             'bool'          :   1,
              'const string&' :   1, } # solves a specific string ctor overload
 
 from rpython.rlib.listsort import make_timsort_class
@@ -414,8 +415,10 @@
 
     def priority(self):
         total_arg_priority = 0
-        for p in [priority.get(arg_type, 0) for arg_type, arg_dflt in self.arg_defs]:
-            total_arg_priority += p
+        for arg_type, arg_dflt in self.arg_defs:
+            total_arg_priority += priority.get(arg_type, 0)
+            if '&&' in arg_type:
+                total_arg_priority += 100
         return total_arg_priority
 
     @rgc.must_be_light_finalizer
@@ -435,7 +438,7 @@
 
 class CPPSetItem(CPPMethod):
     """Method dispatcher specific to Python's __setitem__ mapped onto C++'s
-    operator[](int). The former function takes an extra argument to assign to
+    operator[](T). The former function takes an extra argument to assign to
     the return type of the latter."""
 
     _attrs_ = []
@@ -586,6 +589,14 @@
             sig += '\n'+self.functions[i].prototype()
         return self.space.newtext(sig)
 
+    @unwrap_spec(signature='text')
+    def mp_overload(self, signature):
+        sig = '(%s)' % signature
+        for f in self.functions:
+            if f.signature(False) == sig:
+                return type(self)(self.space, self.scope, [f])
+        raise oefmt(self.space.w_LookupError, "signature \"%s\" not found" % signature)
+
     # allow user to determine ffi use rules per overload
     def fget_useffi(self, space):
         return space.newbool(bool(self.flags & OVERLOAD_FLAGS_USE_FFI))
@@ -609,10 +620,11 @@
 
 W_CPPOverload.typedef = TypeDef(
     'CPPOverload',
-    __get__    = interp2app(W_CPPOverload.descr_get),
-    __call__   = interp2app(W_CPPOverload.call_args),
-    __useffi__ = GetSetProperty(W_CPPOverload.fget_useffi, W_CPPOverload.fset_useffi),
-    __doc__    = GetSetProperty(W_CPPOverload.fget_doc)
+    __get__      = interp2app(W_CPPOverload.descr_get),
+    __call__     = interp2app(W_CPPOverload.call_args),
+    __useffi__   = GetSetProperty(W_CPPOverload.fget_useffi, W_CPPOverload.fset_useffi),
+    __overload__ = interp2app(W_CPPOverload.mp_overload),
+    __doc__      = GetSetProperty(W_CPPOverload.fget_doc)
 )
 
 
@@ -641,10 +653,11 @@
 
 W_CPPStaticOverload.typedef = TypeDef(
     'CPPStaticOverload',
-    __get__    = interp2app(W_CPPStaticOverload.descr_get),
-    __call__   = interp2app(W_CPPStaticOverload.call_args),
-    __useffi__ = GetSetProperty(W_CPPStaticOverload.fget_useffi, W_CPPStaticOverload.fset_useffi),
-    __doc__    = GetSetProperty(W_CPPStaticOverload.fget_doc)
+    __get__      = interp2app(W_CPPStaticOverload.descr_get),
+    __call__     = interp2app(W_CPPStaticOverload.call_args),
+    __useffi__   = GetSetProperty(W_CPPStaticOverload.fget_useffi, W_CPPStaticOverload.fset_useffi),
+    __overload__ = interp2app(W_CPPStaticOverload.mp_overload),
+    __doc__      = GetSetProperty(W_CPPStaticOverload.fget_doc)
 )
 
 
@@ -670,9 +683,10 @@
 
 W_CPPConstructorOverload.typedef = TypeDef(
     'CPPConstructorOverload',
-    __get__    = interp2app(W_CPPConstructorOverload.descr_get),
-    __call__   = interp2app(W_CPPConstructorOverload.call_args),
-    __doc__    = GetSetProperty(W_CPPConstructorOverload.fget_doc)
+    __get__      = interp2app(W_CPPConstructorOverload.descr_get),
+    __call__     = interp2app(W_CPPConstructorOverload.call_args),
+    __overload__ = interp2app(W_CPPConstructorOverload.mp_overload),
+    __doc__      = GetSetProperty(W_CPPConstructorOverload.fget_doc)
 )
 
 class W_CPPAbstractCtorOverload(W_CPPOverload):
@@ -852,11 +866,12 @@
 
 W_CPPTemplateOverload.typedef = TypeDef(
     'CPPTemplateOverload',
-    __get__     = interp2app(W_CPPTemplateOverload.descr_get),
-    __getitem__ = interp2app(W_CPPTemplateOverload.getitem),
-    __call__    = interp2app(W_CPPTemplateOverload.call_args),
-    __useffi__  = GetSetProperty(W_CPPTemplateOverload.fget_useffi, W_CPPTemplateOverload.fset_useffi),
-    __doc__     = GetSetProperty(W_CPPTemplateOverload.fget_doc)
+    __get__      = interp2app(W_CPPTemplateOverload.descr_get),
+    __getitem__  = interp2app(W_CPPTemplateOverload.getitem),
+    __call__     = interp2app(W_CPPTemplateOverload.call_args),
+    __useffi__   = GetSetProperty(W_CPPTemplateOverload.fget_useffi, W_CPPTemplateOverload.fset_useffi),
+    __overload__ = interp2app(W_CPPTemplateOverload.mp_overload),
+    __doc__      = GetSetProperty(W_CPPTemplateOverload.fget_doc)
 )
 
 class W_CPPTemplateStaticOverload(W_CPPStaticOverload, TemplateOverloadMixin):
@@ -910,11 +925,12 @@
 
 W_CPPTemplateStaticOverload.typedef = TypeDef(
     'CPPTemplateStaticOverload',
-    __get__     = interp2app(W_CPPTemplateStaticOverload.descr_get),
-    __getitem__ = interp2app(W_CPPTemplateStaticOverload.getitem),
-    __call__    = interp2app(W_CPPTemplateStaticOverload.call_args),
-    __useffi__  = GetSetProperty(W_CPPTemplateStaticOverload.fget_useffi, W_CPPTemplateStaticOverload.fset_useffi),
-    __doc__     = GetSetProperty(W_CPPTemplateStaticOverload.fget_doc)
+    __get__      = interp2app(W_CPPTemplateStaticOverload.descr_get),
+    __getitem__  = interp2app(W_CPPTemplateStaticOverload.getitem),
+    __call__     = interp2app(W_CPPTemplateStaticOverload.call_args),
+    __useffi__   = GetSetProperty(W_CPPTemplateStaticOverload.fget_useffi, W_CPPTemplateStaticOverload.fset_useffi),
+    __overload__ = interp2app(W_CPPTemplateStaticOverload.mp_overload),
+    __doc__      = GetSetProperty(W_CPPTemplateStaticOverload.fget_doc)
 )
 
 
diff --git a/pypy/module/_cppyy/test/test_overloads.py b/pypy/module/_cppyy/test/test_overloads.py
--- a/pypy/module/_cppyy/test/test_overloads.py
+++ b/pypy/module/_cppyy/test/test_overloads.py
@@ -58,15 +58,19 @@
         c = c_overload()
         raises(TypeError, c.__dispatch__, 'get_int', 12)
         raises(LookupError, c.__dispatch__, 'get_int', 'does_not_exist')
-        assert c.__dispatch__('get_int', 'a_overload*')(a_overload()) == 42
-        assert c.__dispatch__('get_int', 'b_overload*')(b_overload()) == 13
+        assert c.__dispatch__('get_int', 'a_overload*')(a_overload())             == 42
+        assert c_overload.get_int.__overload__('a_overload*')(c, a_overload())    == 42
+        assert c.__dispatch__('get_int', 'b_overload*')(b_overload())             == 13
+        assert c_overload.get_int.__overload__('b_overload*')(c, b_overload())    == 13
 
         assert c_overload().__dispatch__('get_int', 'a_overload*')(a_overload())  == 42
         # TODO: #assert c_overload.__dispatch__('get_int', 'b_overload*')(c, b_overload()) == 13
 
         d = d_overload()
-        assert d.__dispatch__('get_int', 'a_overload*')(a_overload()) == 42
-        assert d.__dispatch__('get_int', 'b_overload*')(b_overload()) == 13
+        assert d.__dispatch__('get_int', 'a_overload*')(a_overload())             == 42
+        assert d_overload.get_int.__overload__('a_overload*')(d, a_overload())    == 42
+        assert d.__dispatch__('get_int', 'b_overload*')(b_overload())             == 13
+        assert d_overload.get_int.__overload__('b_overload*')(d, b_overload())    == 13
 
         nb = ns_a_overload.b_overload()
         raises(TypeError, nb.f, c_overload())

From pypy.commits at gmail.com  Fri Jul 13 02:16:57 2018
From: pypy.commits at gmail.com (wlav)
Date: Thu, 12 Jul 2018 23:16:57 -0700 (PDT)
Subject: [pypy-commit] pypy cppyy-packaging: anotator fixes
Message-ID: <5b4843d9.1c69fb81.28dd.f346@mx.google.com>

Author: Wim Lavrijsen <WLavrijsen at lbl.gov>
Branch: cppyy-packaging
Changeset: r94858:0a2cbe7fb341
Date: 2018-07-12 22:56 -0700
http://bitbucket.org/pypy/pypy/changeset/0a2cbe7fb341/

Log:	anotator fixes

diff --git a/pypy/module/_cppyy/executor.py b/pypy/module/_cppyy/executor.py
--- a/pypy/module/_cppyy/executor.py
+++ b/pypy/module/_cppyy/executor.py
@@ -109,7 +109,7 @@
 
     def _wrap_reference(self, space, rffiptr):
         if self.do_assign:
-            rffiptr[0] = self._unwrap_object(space, self.w_item)
+            rffiptr[0] = rffi.cast(self.c_type, self._unwrap_object(space, self.w_item))
         self.do_assign = False
         return self._wrap_object(space, rffiptr[0])    # all paths, for rtyper
 
diff --git a/pypy/module/_cppyy/interp_cppyy.py b/pypy/module/_cppyy/interp_cppyy.py
--- a/pypy/module/_cppyy/interp_cppyy.py
+++ b/pypy/module/_cppyy/interp_cppyy.py
@@ -594,8 +594,10 @@
         sig = '(%s)' % signature
         for f in self.functions:
             if f.signature(False) == sig:
-                return type(self)(self.space, self.scope, [f])
-        raise oefmt(self.space.w_LookupError, "signature \"%s\" not found" % signature)
+                if isinstance(self, W_CPPStaticOverload):
+                    return W_CPPStaticOverload(self.space, self.scope, [f])
+                return W_CPPOverload(self.space, self.scope, [f])
+        raise oefmt(self.space.w_LookupError, "signature '%s' not found", signature)
 
     # allow user to determine ffi use rules per overload
     def fget_useffi(self, space):
@@ -870,7 +872,6 @@
     __getitem__  = interp2app(W_CPPTemplateOverload.getitem),
     __call__     = interp2app(W_CPPTemplateOverload.call_args),
     __useffi__   = GetSetProperty(W_CPPTemplateOverload.fget_useffi, W_CPPTemplateOverload.fset_useffi),
-    __overload__ = interp2app(W_CPPTemplateOverload.mp_overload),
     __doc__      = GetSetProperty(W_CPPTemplateOverload.fget_doc)
 )
 
@@ -929,7 +930,6 @@
     __getitem__  = interp2app(W_CPPTemplateStaticOverload.getitem),
     __call__     = interp2app(W_CPPTemplateStaticOverload.call_args),
     __useffi__   = GetSetProperty(W_CPPTemplateStaticOverload.fget_useffi, W_CPPTemplateStaticOverload.fset_useffi),
-    __overload__ = interp2app(W_CPPTemplateStaticOverload.mp_overload),
     __doc__      = GetSetProperty(W_CPPTemplateStaticOverload.fget_doc)
 )
 
@@ -1103,7 +1103,9 @@
         sig = '(%s)' % signature
         for f in overload.functions:
             if f.signature(False) == sig:
-                return type(overload)(self.space, self, [f])
+                if isinstance(overload, W_CPPStaticOverload):
+                    return W_CPPStaticOverload(self.space, self, [f])
+                return W_CPPOverload(self.space, self, [f])
         raise oefmt(self.space.w_LookupError, "no overload matches signature")
 
     def __eq__(self, other):

From pypy.commits at gmail.com  Fri Jul 13 15:14:34 2018
From: pypy.commits at gmail.com (wlav)
Date: Fri, 13 Jul 2018 12:14:34 -0700 (PDT)
Subject: [pypy-commit] pypy cppyy-packaging: some more asserts for the
 annotator (gives a small performance improvement on bound function calls)
Message-ID: <5b48fa1a.1c69fb81.61c1a.c9c0@mx.google.com>

Author: Wim Lavrijsen <WLavrijsen at lbl.gov>
Branch: cppyy-packaging
Changeset: r94859:a3af4486e271
Date: 2018-07-13 09:29 -0700
http://bitbucket.org/pypy/pypy/changeset/a3af4486e271/

Log:	some more asserts for the annotator (gives a small performance
	improvement on bound function calls)

diff --git a/pypy/module/_cppyy/interp_cppyy.py b/pypy/module/_cppyy/interp_cppyy.py
--- a/pypy/module/_cppyy/interp_cppyy.py
+++ b/pypy/module/_cppyy/interp_cppyy.py
@@ -958,6 +958,7 @@
 
     def _get_offset(self, cppinstance):
         if cppinstance:
+            assert isinstance(cppinstance.clsdecl, W_CPPClassDecl)
             assert lltype.typeOf(cppinstance.clsdecl.handle) == lltype.typeOf(self.scope.handle)
             offset = self.offset + cppinstance.clsdecl.get_base_offset(cppinstance, self.scope)
         else:
@@ -1314,10 +1315,12 @@
         raise self.missing_attribute_error(name)
 
     def get_base_offset(self, cppinstance, calling_scope):
+        assert isinstance(cppinstance.clsdecl, W_CPPClassDecl)
         assert self == cppinstance.clsdecl
         return 0
 
     def get_cppthis(self, cppinstance, calling_scope):
+        assert isinstance(cppinstance.clsdecl, W_CPPClassDecl)
         assert self == cppinstance.clsdecl
         return cppinstance.get_rawobject()
 
@@ -1353,12 +1356,14 @@
 
 class W_CPPComplexClassDecl(W_CPPClassDecl):
     def get_base_offset(self, cppinstance, calling_scope):
+        assert isinstance(cppinstance.clsdecl, W_CPPComplexClassDecl)
         assert self == cppinstance.clsdecl
         offset = capi.c_base_offset(self.space,
             self, calling_scope, cppinstance.get_rawobject(), 1)
         return offset
 
     def get_cppthis(self, cppinstance, calling_scope):
+        assert isinstance(cppinstance.clsdecl, W_CPPComplexClassDecl)
         assert self == cppinstance.clsdecl
         offset = self.get_base_offset(cppinstance, calling_scope)
         return capi.direct_ptradd(cppinstance.get_rawobject(), offset)
@@ -1388,6 +1393,7 @@
                  smartdecl=None, deref=rffi.cast(capi.C_METHOD, 0)):
         self.space = space
         self.clsdecl = decl
+        assert isinstance(self.clsdecl, W_CPPClassDecl)
         assert lltype.typeOf(rawobject) == capi.C_OBJECT
         assert not isref or rawobject
         self._rawobject = rawobject
@@ -1427,6 +1433,7 @@
             self.rt_flags &= ~INSTANCE_FLAGS_PYTHON_OWNS
 
     def get_cppthis(self, calling_scope):
+        assert isinstance(self.clsdecl, W_CPPClassDecl)
         return self.clsdecl.get_cppthis(self, calling_scope)
 
     def get_rawobject(self):
@@ -1533,6 +1540,7 @@
 
     def destruct(self):
         if self._rawobject:
+            assert isinstance(self.clsdecl, W_CPPClassDecl)
             if self.smartdecl and self.deref:
                 klass = self.smartdecl
             elif not (self.flags & INSTANCE_FLAGS_IS_REF):

From pypy.commits at gmail.com  Fri Jul 13 15:14:36 2018
From: pypy.commits at gmail.com (wlav)
Date: Fri, 13 Jul 2018 12:14:36 -0700 (PDT)
Subject: [pypy-commit] pypy cppyy-packaging: support low-level views of
 arrays that allow user-side updates to shape
Message-ID: <5b48fa1c.1c69fb81.1b755.1b8c@mx.google.com>

Author: Wim Lavrijsen <WLavrijsen at lbl.gov>
Branch: cppyy-packaging
Changeset: r94860:1214d76af168
Date: 2018-07-13 11:44 -0700
http://bitbucket.org/pypy/pypy/changeset/1214d76af168/

Log:	support low-level views of arrays that allow user-side updates to
	shape

diff --git a/pypy/module/_cppyy/converter.py b/pypy/module/_cppyy/converter.py
--- a/pypy/module/_cppyy/converter.py
+++ b/pypy/module/_cppyy/converter.py
@@ -1,15 +1,12 @@
 import sys
 
 from pypy.interpreter.error import OperationError, oefmt
-
 from rpython.rtyper.lltypesystem import rffi, lltype
 from rpython.rlib.rarithmetic import r_singlefloat, r_longfloat
 from rpython.rlib import rfloat, rawrefcount
-
 from pypy.module._rawffi.interp_rawffi import letter2tp
 from pypy.module._rawffi.array import W_ArrayInstance
-
-from pypy.module._cppyy import helper, capi, ffitypes
+from pypy.module._cppyy import helper, capi, ffitypes, lowlevelviews
 
 # Converter objects are used to translate between RPython and C++. They are
 # defined by the type name for which they provide conversion. Uses are for
@@ -149,7 +146,8 @@
         # read access, so no copy needed
         address_value = self._get_raw_address(space, w_obj, offset)
         address = rffi.cast(rffi.ULONG, address_value)
-        return W_ArrayInstance(space, letter2tp(space, self.typecode), self.size, address)
+        return lowlevelviews.W_LowLevelView(
+            space, letter2tp(space, self.typecode), self.size, address)
 
     def to_memory(self, space, w_obj, w_value, offset):
         # copy the full array (uses byte copy for now)
@@ -190,7 +188,8 @@
         # read access, so no copy needed
         address_value = self._get_raw_address(space, w_obj, offset)
         address = rffi.cast(rffi.ULONGP, address_value)
-        return W_ArrayInstance(space, letter2tp(space, self.typecode), self.size, address[0])
+        return lowlevelviews.W_LowLevelView(
+            space, letter2tp(space, self.typecode), self.size, address[0])
 
     def to_memory(self, space, w_obj, w_value, offset):
         # copy only the pointer value
@@ -438,7 +437,7 @@
             from pypy.module._cppyy import interp_cppyy
             return interp_cppyy.get_nullptr(space)
         shape = letter2tp(space, 'P')
-        return W_ArrayInstance(space, shape, sys.maxint/shape.size, ptrval)
+        return lowlevelviews.W_LowLevelView(space, shape, sys.maxint/shape.size, ptrval)
 
     def to_memory(self, space, w_obj, w_value, offset):
         address = rffi.cast(rffi.VOIDPP, self._get_raw_address(space, w_obj, offset))
diff --git a/pypy/module/_cppyy/executor.py b/pypy/module/_cppyy/executor.py
--- a/pypy/module/_cppyy/executor.py
+++ b/pypy/module/_cppyy/executor.py
@@ -1,14 +1,10 @@
 import sys
 
 from pypy.interpreter.error import oefmt
-
 from rpython.rtyper.lltypesystem import rffi, lltype
 from rpython.rlib import jit_libffi
-
 from pypy.module._rawffi.interp_rawffi import letter2tp
-from pypy.module._rawffi.array import W_Array, W_ArrayInstance
-
-from pypy.module._cppyy import helper, capi, ffitypes
+from pypy.module._cppyy import helper, capi, ffitypes, lowlevelviews
 
 # Executor objects are used to dispatch C++ methods. They are defined by their
 # return type only: arguments are converted by Converter objects, and Executors
@@ -60,7 +56,7 @@
             from pypy.module._cppyy import interp_cppyy
             return interp_cppyy.get_nullptr(space)
         shape = letter2tp(space, self.typecode)
-        return W_ArrayInstance(space, shape, sys.maxint/shape.size, ptrval)
+        return lowlevelviews.W_LowLevelView(space, shape, sys.maxint/shape.size, ptrval)
 
 
 class VoidExecutor(Executor):
diff --git a/pypy/module/_cppyy/lowlevelviews.py b/pypy/module/_cppyy/lowlevelviews.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/_cppyy/lowlevelviews.py
@@ -0,0 +1,54 @@
+# Naked C++ pointers carry no useful size or layout information, but often
+# such information is externnally available. Low level views are arrays with
+# a few more methods allowing such information to be set. Afterwards, it is
+# simple to pass these views on to e.g. numpy (w/o the need to copy).
+
+from pypy.interpreter.error import oefmt
+from pypy.interpreter.gateway import interp2app, unwrap_spec
+from pypy.interpreter.typedef import TypeDef, GetSetProperty, interp_attrproperty_w
+from pypy.module._rawffi.array import W_ArrayInstance
+
+
+class W_LowLevelView(W_ArrayInstance):
+    def __init__(self, space, shape, length, address):
+        assert address   # if not address, base class will allocate memory
+        W_ArrayInstance.__init__(self, space, shape, length, address)
+
+    @unwrap_spec(args_w='args_w')
+    def reshape(self, space, args_w):
+        # llviews are only created from non-zero addresses, so we only need
+        # to adjust length and shape
+
+        nargs = len(args_w)
+        if nargs == 0:
+            raise oefmt(space.w_TypeError, "reshape expects a tuple argument")
+
+        newshape_w = args_w
+        if nargs != 1 or not space.isinstance_w(args_w[0], space.w_tuple) or \
+               not space.len_w(args_w[0]) == 1:
+            raise oefmt(space.w_TypeError,
+                "tuple object of length 1 expected, received %T", args_w[0])
+
+        w_shape = args_w[0]
+
+        # shape in W_ArrayInstance-speak is somewhat different from what
+        # e.g. numpy thinks of it: self.shape contains the info (itemcode,
+        # size, etc.) of a single entry; length is user-facing shape
+        self.length = space.int_w(space.getitem(w_shape, space.newint(0)))
+
+
+W_LowLevelView.typedef = TypeDef(
+    'LowLevelView',
+    __repr__    = interp2app(W_LowLevelView.descr_repr),
+    __setitem__ = interp2app(W_LowLevelView.descr_setitem),
+    __getitem__ = interp2app(W_LowLevelView.descr_getitem),
+    __len__     = interp2app(W_LowLevelView.getlength),
+    buffer      = GetSetProperty(W_LowLevelView.getbuffer),
+    shape       = interp_attrproperty_w('shape', W_LowLevelView),
+    free        = interp2app(W_LowLevelView.free),
+    byptr       = interp2app(W_LowLevelView.byptr),
+    itemaddress = interp2app(W_LowLevelView.descr_itemaddress),
+    reshape     = interp2app(W_LowLevelView.reshape),
+)
+W_ArrayInstance.typedef.acceptable_as_base_class = False
+
diff --git a/pypy/module/_cppyy/test/test_datatypes.py b/pypy/module/_cppyy/test/test_datatypes.py
--- a/pypy/module/_cppyy/test/test_datatypes.py
+++ b/pypy/module/_cppyy/test/test_datatypes.py
@@ -181,7 +181,7 @@
         names = ['uchar', 'short', 'ushort', 'int', 'uint', 'long', 'ulong']
         import array
         a = range(self.N)
-        atypes = ['B', 'h', 'H', 'i', 'I', 'l', 'L' ]
+        atypes = ['B', 'h', 'H', 'i', 'I', 'l', 'L']
         for j in range(len(names)):
             b = array.array(atypes[j], a)
             setattr(c, 'm_'+names[j]+'_array', b)     # buffer copies
@@ -189,6 +189,7 @@
                 assert eval('c.m_%s_array[i]' % names[j]) == b[i]
 
             setattr(c, 'm_'+names[j]+'_array2', b)    # pointer copies
+            assert 3 < self.N
             b[3] = 28
             for i in range(self.N):
                 assert eval('c.m_%s_array2[i]' % names[j]) == b[i]
@@ -210,7 +211,7 @@
 
         a = range(self.N)
         # test arrays in mixed order, to give overload resolution a workout
-        for t in ['d', 'i', 'f', 'H', 'I', 'h', 'L', 'l' ]:
+        for t in ['d', 'i', 'f', 'H', 'I', 'h', 'L', 'l']:
             b = array.array(t, a)
 
             # typed passing
@@ -689,9 +690,6 @@
                      'get_long_array',   'get_long_array2',
                      'get_ulong_array',  'get_ulong_array2']:
             arr = getattr(c, func)()
-            arr = arr.shape.fromaddress(arr.itemaddress(0), self.N)
-
-            """TODO: interface change ...
             arr.reshape((self.N,))
             assert len(arr) == self.N
 
@@ -703,7 +701,7 @@
 
             l = list(arr)
             for i in range(self.N):
-                assert arr[i] == l[i]"""
+                assert arr[i] == l[i]
 
     def test20_voidp(self):
         """Test usage of void* data"""
@@ -756,6 +754,15 @@
     def test21_function_pointers(self):
         """Function pointer passing"""
 
+        import os
+
+        # TODO: currently crashes if fast path disabled
+        try:
+            if os.environ['CPPYY_DISABLE_FASTPATH']:
+                return
+        except KeyError:
+            pass
+
         import _cppyy as cppyy
 
         f1 = cppyy.gbl.sum_of_int

From pypy.commits at gmail.com  Fri Jul 13 15:14:38 2018
From: pypy.commits at gmail.com (wlav)
Date: Fri, 13 Jul 2018 12:14:38 -0700 (PDT)
Subject: [pypy-commit] pypy cppyy-packaging: more pythonizatin of
 std::vector and more STL tests
Message-ID: <5b48fa1e.1c69fb81.b94ae.d97c@mx.google.com>

Author: Wim Lavrijsen <WLavrijsen at lbl.gov>
Branch: cppyy-packaging
Changeset: r94861:dbc2035e00e4
Date: 2018-07-13 11:44 -0700
http://bitbucket.org/pypy/pypy/changeset/dbc2035e00e4/

Log:	more pythonizatin of std::vector and more STL tests

diff --git a/pypy/module/_cppyy/pythonify.py b/pypy/module/_cppyy/pythonify.py
--- a/pypy/module/_cppyy/pythonify.py
+++ b/pypy/module/_cppyy/pythonify.py
@@ -451,6 +451,14 @@
             return self.__real_init__(*args)
         pyclass.__init__ = vector_init
 
+        # size-up the return of data()
+        pyclass.__real_data = pyclass.data
+        def data_with_len(self):
+            arr = self.__real_data()
+            arr.reshape((len(self),))
+            return arr
+        pyclass.data = data_with_len
+
     # combine __getitem__ and __len__ to make a pythonized __getitem__
     if '__getitem__' in pyclass.__dict__ and '__len__' in pyclass.__dict__:
         pyclass._getitem__unchecked = pyclass.__getitem__
diff --git a/pypy/module/_cppyy/test/stltypes.cxx b/pypy/module/_cppyy/test/stltypes.cxx
--- a/pypy/module/_cppyy/test/stltypes.cxx
+++ b/pypy/module/_cppyy/test/stltypes.cxx
@@ -1,6 +1,20 @@
 #include "stltypes.h"
 
 
+//- explicit instantiations of used comparisons
+#if defined __clang__
+namespace std {
+#define ns_prefix std::
+#elif defined(__GNUC__) || defined(__GNUG__)
+namespace __gnu_cxx {
+#define ns_prefix
+#endif
+template bool ns_prefix operator==(const std::vector<int>::iterator&,
+                         const std::vector<int>::iterator&);
+template bool ns_prefix operator!=(const std::vector<int>::iterator&,
+                         const std::vector<int>::iterator&);
+}
+
 //- class with lots of std::string handling
 stringy_class::stringy_class(const char* s) : m_string(s) {}
 
@@ -9,3 +23,33 @@
 
 void stringy_class::set_string1(const std::string& s) { m_string = s; }
 void stringy_class::set_string2(std::string s) { m_string = s; }
+
+
+//- helpers for testing array
+int ArrayTest::get_pp_px(Point** p, int idx) {
+    return p[idx]->px;
+}
+
+int ArrayTest::get_pp_py(Point** p, int idx) {
+    return p[idx]->py;
+}
+
+int ArrayTest::get_pa_px(Point* p[], int idx) {
+    return p[idx]->px;
+}
+
+int ArrayTest::get_pa_py(Point* p[], int idx) {
+    return p[idx]->py;
+}
+
+
+// helpers for string testing
+std::string str_array_1[3] = {"a", "b", "c"};
+std::string str_array_2[]  = {"d", "e", "f", "g"};
+std::string str_array_3[3][2] = {{"a", "b"}, {"c", "d"}, {"e", "f"}};
+std::string str_array_4[4][2][2] = {
+     {{"a", "b"}, {"c", "d"}},
+     {{"e", "f"}, {"g", "h"}},
+     {{"i", "j"}, {"k", "l"}},
+     {{"m", "n"}, {"o", "p"}},
+};
diff --git a/pypy/module/_cppyy/test/stltypes.h b/pypy/module/_cppyy/test/stltypes.h
--- a/pypy/module/_cppyy/test/stltypes.h
+++ b/pypy/module/_cppyy/test/stltypes.h
@@ -37,3 +37,49 @@
    std::string operator[](double) { return "double"; }
    std::string operator[](const std::string&) { return "string"; }
 };      
+
+
+//- instantiations of used STL types
+namespace {
+
+    stl_like_class<int> stlc_1;
+
+} // unnamed namespace
+
+
+// comps for int only to allow testing: normal use of vector is looping over a
+// range-checked version of __getitem__
+#if defined(__clang__) && defined(__APPLE__)
+namespace std {
+#define ns_prefix std::
+#elif defined(__GNUC__) || defined(__GNUG__)
+namespace __gnu_cxx {
+#define ns_prefix
+#endif
+extern template bool ns_prefix operator==(const std::vector<int>::iterator&,
+                         const std::vector<int>::iterator&);
+extern template bool ns_prefix operator!=(const std::vector<int>::iterator&,
+                         const std::vector<int>::iterator&);
+}
+
+
+//- helpers for testing array
+namespace ArrayTest {
+
+struct Point {
+    int px, py;
+};
+
+int get_pp_px(Point** p, int idx);
+int get_pp_py(Point** p, int idx);
+int get_pa_px(Point* p[], int idx);
+int get_pa_py(Point* p[], int idx);
+
+} // namespace ArrayTest
+
+
+// helpers for string testing
+extern std::string str_array_1[3];
+extern std::string str_array_2[];
+extern std::string str_array_3[3][2];
+extern std::string str_array_4[4][2][2];
diff --git a/pypy/module/_cppyy/test/stltypes.xml b/pypy/module/_cppyy/test/stltypes.xml
--- a/pypy/module/_cppyy/test/stltypes.xml
+++ b/pypy/module/_cppyy/test/stltypes.xml
@@ -1,10 +1,13 @@
 <lcgdict>
 
-  <namespace name="std" />
+  <class name="just_a_class" />
+  <class name="stringy_class" />
+  <class pattern="stl_like_class<*>" />
 
-  <class name="just_a_class" />
+  <namespace name="ArrayTest" />
+  <class pattern="ArrayTest::*" />
+  <function pattern="ArrayTest::*" />
 
-  <class name="std::string" />
-  <class name="stringy_class" />
+  <variable pattern="str_array_*" />
 
 </lcgdict>
diff --git a/pypy/module/_cppyy/test/test_stltypes.py b/pypy/module/_cppyy/test/test_stltypes.py
--- a/pypy/module/_cppyy/test/test_stltypes.py
+++ b/pypy/module/_cppyy/test/test_stltypes.py
@@ -42,8 +42,8 @@
             assert tv1.iterator is cppyy.gbl.std.vector(p_type).iterator
 
             #----- 
-            v = tv1(); v += range(self.N)    # default args from Reflex are useless :/
-            if p_type == int:                # only type with == and != reflected in .xml
+            v = tv1(); v += range(self.N)
+            if p_type == int:
                 assert v.begin().__eq__(v.begin())
                 assert v.begin() == v.begin()
                 assert v.end() == v.end()
@@ -58,6 +58,7 @@
 
             assert v.size() == self.N
             assert len(v) == self.N
+            assert len(v.data()) == self.N
 
             #-----
             v = tv1()
@@ -502,3 +503,54 @@
         assert len(v) == N
         for i in range(N):
             assert v[i] == i
+
+
+class AppTestSTLARRAY:
+    spaceconfig = dict(usemodules=['_cppyy', '_rawffi', 'itertools'])
+
+    def setup_class(cls):
+        cls.w_test_dct  = cls.space.newtext(test_dct)
+        cls.w_stlarray = cls.space.appexec([], """():
+            import ctypes, _cppyy
+            _cppyy._post_import_startup()
+            return ctypes.CDLL(%r, ctypes.RTLD_GLOBAL)""" % (test_dct, ))
+
+    def test01_array_of_basic_types(self):
+        """Usage of std::array of basic types"""
+
+        import _cppyy as cppyy
+        std = cppyy.gbl.std
+
+        a = std.array[int, 4]()
+        assert len(a) == 4
+        for i in range(len(a)):
+            a[i] = i
+            assert a[i] == i
+
+    def test02_array_of_pods(self):
+        """Usage of std::array of PODs"""
+
+        import _cppyy as cppyy
+        gbl, std = cppyy.gbl, cppyy.gbl.std
+
+        a = std.array[gbl.ArrayTest.Point, 4]()
+        assert len(a) == 4
+        for i in range(len(a)):
+            a[i].px = i
+            assert a[i].px == i
+            a[i].py = i**2
+            assert a[i].py == i**2
+
+    def test03_array_of_pointer_to_pods(self):
+        """Usage of std::array of pointer to PODs"""
+
+        import cppyy
+        from cppyy import gbl
+        from cppyy.gbl import std
+
+        ll = [gbl.ArrayTest.Point() for i in range(4)]
+        for i in range(len(ll)):
+            ll[i].px = 13*i
+            ll[i].py = 42*i
+
+        # more tests in cppyy/test/test_stltypes.py, but currently not supported

From pypy.commits at gmail.com  Sat Jul 14 02:37:57 2018
From: pypy.commits at gmail.com (wlav)
Date: Fri, 13 Jul 2018 23:37:57 -0700 (PDT)
Subject: [pypy-commit] pypy cppyy-packaging: do not use string default/error
 size as object size
Message-ID: <5b499a45.1c69fb81.8a162.4875@mx.google.com>

Author: Wim Lavrijsen <WLavrijsen at lbl.gov>
Branch: cppyy-packaging
Changeset: r94865:7c3f397e2434
Date: 2018-07-13 23:17 -0700
http://bitbucket.org/pypy/pypy/changeset/7c3f397e2434/

Log:	do not use string default/error size as object size

diff --git a/pypy/module/_cppyy/converter.py b/pypy/module/_cppyy/converter.py
--- a/pypy/module/_cppyy/converter.py
+++ b/pypy/module/_cppyy/converter.py
@@ -400,10 +400,12 @@
     def from_memory(self, space, w_obj, w_pycppclass, offset):
         address = self._get_raw_address(space, w_obj, offset)
         charpptr = rffi.cast(rffi.CCHARP, address)
-        strsize = self.size
-        if charpptr[self.size-1] == '\0':
-            strsize = self.size-1  # rffi will add \0 back
-        return space.newtext(rffi.charpsize2str(charpptr, strsize))
+        if 0 <= self.size and self.size != 2**31-1:   # cling's code for "unknown" (?)
+            strsize = self.size
+            if charpptr[self.size-1] == '\0':
+                strsize = self.size-1  # rffi will add \0 back
+            return space.newtext(rffi.charpsize2str(charpptr, strsize))
+        return space.newtext(rffi.charp2str(charpptr))
 
 
 class VoidPtrConverter(TypeConverter):

From pypy.commits at gmail.com  Sat Jul 14 03:47:49 2018
From: pypy.commits at gmail.com (arigo)
Date: Sat, 14 Jul 2018 00:47:49 -0700 (PDT)
Subject: [pypy-commit] pypy default: Issue #2857
Message-ID: <5b49aaa5.1c69fb81.a0751.5e9e@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r94866:fb05c07c73c5
Date: 2018-07-14 09:46 +0200
http://bitbucket.org/pypy/pypy/changeset/fb05c07c73c5/

Log:	Issue #2857

	Fix .casefold() in some cases, for py3.5

diff --git a/rpython/rlib/unicodedata/generate_unicodedb.py b/rpython/rlib/unicodedata/generate_unicodedb.py
--- a/rpython/rlib/unicodedata/generate_unicodedb.py
+++ b/rpython/rlib/unicodedata/generate_unicodedb.py
@@ -913,8 +913,17 @@
 
     casefolds = {}
     for code, char in table.enum_chars():
-        if char.casefolding and char.casefolding != [char.lower]:
-            casefolds[code] = char.casefolding
+        full_casefold = char.casefolding
+        if full_casefold is None:
+            full_casefold = [code]
+        full_lower = char.lower
+        if full_lower is None:
+            full_lower = code
+        # if we don't write anything into the file, then the RPython
+        # program would compute the result 'full_lower' instead.
+        # Is that the right answer?
+        if full_casefold != [full_lower]:
+            casefolds[code] = full_casefold
     writeDict(outfile, '_casefolds', casefolds, base_mod)
     print >> outfile, '''
 
diff --git a/rpython/rlib/unicodedata/test/test_unicodedata.py b/rpython/rlib/unicodedata/test/test_unicodedata.py
--- a/rpython/rlib/unicodedata/test/test_unicodedata.py
+++ b/rpython/rlib/unicodedata/test/test_unicodedata.py
@@ -148,3 +148,15 @@
     def test_changed_in_version_8(self):
         assert unicodedb_6_2_0.toupper_full(0x025C) == [0x025C]
         assert unicodedb_8_0_0.toupper_full(0x025C) == [0xA7AB]
+
+    def test_casefold(self):
+        # returns None when we have no special casefolding rule,
+        # which means that tolower_full() should be used instead
+        assert unicodedb_8_0_0.casefold_lookup(0x1000) == None
+        assert unicodedb_8_0_0.casefold_lookup(0x0061) == None
+        assert unicodedb_8_0_0.casefold_lookup(0x0041) == None
+        # a case where casefold() != lower()
+        assert unicodedb_8_0_0.casefold_lookup(0x00DF) == [ord('s'), ord('s')]
+        # returns the argument itself, and not None, in rare cases
+        # where tolower_full() would return something different
+        assert unicodedb_8_0_0.casefold_lookup(0x13A0) == [0x13A0]
diff --git a/rpython/rlib/unicodedata/unicodedb_8_0_0.py b/rpython/rlib/unicodedata/unicodedb_8_0_0.py
--- a/rpython/rlib/unicodedata/unicodedb_8_0_0.py
+++ b/rpython/rlib/unicodedata/unicodedb_8_0_0.py
@@ -21307,6 +21307,92 @@
         return code
 
 _casefolds = {
+5024: [5024],
+5025: [5025],
+5026: [5026],
+5027: [5027],
+5028: [5028],
+5029: [5029],
+5030: [5030],
+5031: [5031],
+5032: [5032],
+5033: [5033],
+5034: [5034],
+5035: [5035],
+5036: [5036],
+5037: [5037],
+5038: [5038],
+5039: [5039],
+5040: [5040],
+5041: [5041],
+5042: [5042],
+5043: [5043],
+5044: [5044],
+5045: [5045],
+5046: [5046],
+5047: [5047],
+5048: [5048],
+5049: [5049],
+5050: [5050],
+5051: [5051],
+5052: [5052],
+5053: [5053],
+5054: [5054],
+5055: [5055],
+5056: [5056],
+5057: [5057],
+5058: [5058],
+5059: [5059],
+5060: [5060],
+5061: [5061],
+5062: [5062],
+5063: [5063],
+5064: [5064],
+5065: [5065],
+5066: [5066],
+5067: [5067],
+5068: [5068],
+5069: [5069],
+5070: [5070],
+5071: [5071],
+5072: [5072],
+5073: [5073],
+5074: [5074],
+5075: [5075],
+5076: [5076],
+5077: [5077],
+5078: [5078],
+5079: [5079],
+5080: [5080],
+5081: [5081],
+5082: [5082],
+5083: [5083],
+5084: [5084],
+5085: [5085],
+5086: [5086],
+5087: [5087],
+5088: [5088],
+5089: [5089],
+5090: [5090],
+5091: [5091],
+5092: [5092],
+5093: [5093],
+5094: [5094],
+5095: [5095],
+5096: [5096],
+5097: [5097],
+5098: [5098],
+5099: [5099],
+5100: [5100],
+5101: [5101],
+5102: [5102],
+5103: [5103],
+5104: [5104],
+5105: [5105],
+5106: [5106],
+5107: [5107],
+5108: [5108],
+5109: [5109],
 5112: [5104],
 5113: [5105],
 5114: [5106],
diff --git a/rpython/rlib/unicodedata/unicodedb_9_0_0.py b/rpython/rlib/unicodedata/unicodedb_9_0_0.py
--- a/rpython/rlib/unicodedata/unicodedb_9_0_0.py
+++ b/rpython/rlib/unicodedata/unicodedb_9_0_0.py
@@ -24430,6 +24430,92 @@
         return code
 
 _casefolds = {
+5024: [5024],
+5025: [5025],
+5026: [5026],
+5027: [5027],
+5028: [5028],
+5029: [5029],
+5030: [5030],
+5031: [5031],
+5032: [5032],
+5033: [5033],
+5034: [5034],
+5035: [5035],
+5036: [5036],
+5037: [5037],
+5038: [5038],
+5039: [5039],
+5040: [5040],
+5041: [5041],
+5042: [5042],
+5043: [5043],
+5044: [5044],
+5045: [5045],
+5046: [5046],
+5047: [5047],
+5048: [5048],
+5049: [5049],
+5050: [5050],
+5051: [5051],
+5052: [5052],
+5053: [5053],
+5054: [5054],
+5055: [5055],
+5056: [5056],
+5057: [5057],
+5058: [5058],
+5059: [5059],
+5060: [5060],
+5061: [5061],
+5062: [5062],
+5063: [5063],
+5064: [5064],
+5065: [5065],
+5066: [5066],
+5067: [5067],
+5068: [5068],
+5069: [5069],
+5070: [5070],
+5071: [5071],
+5072: [5072],
+5073: [5073],
+5074: [5074],
+5075: [5075],
+5076: [5076],
+5077: [5077],
+5078: [5078],
+5079: [5079],
+5080: [5080],
+5081: [5081],
+5082: [5082],
+5083: [5083],
+5084: [5084],
+5085: [5085],
+5086: [5086],
+5087: [5087],
+5088: [5088],
+5089: [5089],
+5090: [5090],
+5091: [5091],
+5092: [5092],
+5093: [5093],
+5094: [5094],
+5095: [5095],
+5096: [5096],
+5097: [5097],
+5098: [5098],
+5099: [5099],
+5100: [5100],
+5101: [5101],
+5102: [5102],
+5103: [5103],
+5104: [5104],
+5105: [5105],
+5106: [5106],
+5107: [5107],
+5108: [5108],
+5109: [5109],
 5112: [5104],
 5113: [5105],
 5114: [5106],

From pypy.commits at gmail.com  Sat Jul 14 03:47:52 2018
From: pypy.commits at gmail.com (arigo)
Date: Sat, 14 Jul 2018 00:47:52 -0700 (PDT)
Subject: [pypy-commit] pypy default: merge heads
Message-ID: <5b49aaa8.1c69fb81.6eb12.5a21@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r94867:46410c58b280
Date: 2018-07-14 09:47 +0200
http://bitbucket.org/pypy/pypy/changeset/46410c58b280/

Log:	merge heads

diff --git a/pypy/doc/config/objspace.disable_entrypoints.txt b/pypy/doc/config/objspace.disable_entrypoints.txt
new file mode 100644
diff --git a/pypy/doc/config/objspace.fstrings.txt b/pypy/doc/config/objspace.fstrings.txt
new file mode 100644
diff --git a/pypy/doc/config/objspace.hash.txt b/pypy/doc/config/objspace.hash.txt
new file mode 100644
diff --git a/pypy/doc/config/objspace.usemodules._frozen_importlib.txt b/pypy/doc/config/objspace.usemodules._frozen_importlib.txt
new file mode 100644
diff --git a/pypy/doc/config/objspace.usemodules._jitlog.txt b/pypy/doc/config/objspace.usemodules._jitlog.txt
new file mode 100644
diff --git a/pypy/doc/config/objspace.usemodules.faulthandler.txt b/pypy/doc/config/objspace.usemodules.faulthandler.txt
new file mode 100644
diff --git a/pypy/doc/config/translation.backendopt.replace_we_are_jitted.txt b/pypy/doc/config/translation.backendopt.replace_we_are_jitted.txt
new file mode 100644
diff --git a/pypy/doc/config/translation.jit_opencoder_model.txt b/pypy/doc/config/translation.jit_opencoder_model.txt
new file mode 100644
diff --git a/pypy/doc/config/translation.keepgoing.txt b/pypy/doc/config/translation.keepgoing.txt
new file mode 100644
diff --git a/pypy/doc/config/translation.libname.txt b/pypy/doc/config/translation.libname.txt
new file mode 100644
diff --git a/pypy/doc/config/translation.lto.txt b/pypy/doc/config/translation.lto.txt
new file mode 100644
diff --git a/pypy/doc/config/translation.profoptargs.txt b/pypy/doc/config/translation.profoptargs.txt
new file mode 100644
diff --git a/pypy/doc/config/translation.reverse_debugger.txt b/pypy/doc/config/translation.reverse_debugger.txt
new file mode 100644
diff --git a/pypy/doc/config/translation.split_gc_address_space.txt b/pypy/doc/config/translation.split_gc_address_space.txt
new file mode 100644
diff --git a/pypy/doc/tool/makecontributor.py b/pypy/doc/tool/makecontributor.py
--- a/pypy/doc/tool/makecontributor.py
+++ b/pypy/doc/tool/makecontributor.py
@@ -18,12 +18,13 @@
     'Antonio Cuni': ['antocuni', 'anto'],
     'Armin Rigo': ['arigo', 'arfigo', 'armin', 'arigato'],
     'Maciej Fijalkowski': ['fijal'],
-    'Carl Friedrich Bolz-Tereick': ['Carl Friedrich Bolz', 'cfbolz', 'cf'],
+    'Carl Friedrich Bolz-Tereick': ['Carl Friedrich Bolz', 'cfbolz', 'cf', 'cbolz'],
     'Samuele Pedroni': ['pedronis', 'samuele', 'samule'],
-    'Richard Plangger':['planrich'],
-    'Michael Hudson': ['mwh'],
+    'Richard Plangger': ['planrich', 'plan_rich'],
+    'Remi Meier': ['remi'],
+    'Michael Hudson-Doyle': ['mwh', 'Michael Hudson'],
     'Holger Krekel': ['hpk', 'holger krekel', 'holger', 'hufpk'],
-    "Amaury Forgeot d'Arc": ['afa'],
+    "Amaury Forgeot d'Arc": ['afa', 'amauryfa at gmail.com'],
     'Alex Gaynor': ['alex', 'agaynor'],
     'David Schneider': ['bivab', 'david'],
     'Christian Tismer': ['chris', 'christian', 'tismer',
@@ -41,7 +42,7 @@
     'Mark Pearse': ['mwp'],
     'Toon Verwaest': ['tverwaes'],
     'Eric van Riet Paap': ['ericvrp'],
-    'Jacob Hallen': ['jacob', 'jakob'],
+    'Jacob Hallen': ['jacob', 'jakob', 'jacob hallen'],
     'Anders Lehmann': ['ale', 'anders'],
     'Bert Freudenberg': ['bert'],
     'Boris Feigin': ['boris', 'boria'],
@@ -69,19 +70,25 @@
     'Manuel Jacob': ['mjacob'],
     'Rami Chowdhury': ['necaris'],
     'Stanislaw Halik': ['Stanislaw Halik', 'w31rd0'],
-    'Wenzhu Man':['wenzhu man', 'wenzhuman'],
-    'Anton Gulenko':['anton gulenko', 'anton_gulenko'],
-    'Richard Lancaster':['richardlancaster'],
-    'William Leslie':['William ML Leslie'],
-    'Spenser Bauman':['Spenser Andrew Bauman'],
-    'Raffael Tfirst':['raffael.tfirst at gmail.com'],
-    'timo':['timo at eistee.fritz.box'],
-    'Jasper Schulz':['Jasper.Schulz', 'jbs'],
-    'Aaron Gallagher':['"Aaron Gallagher'],
-    'Yasir Suhail':['yasirs'],
+    'Wenzhu Man': ['wenzhu man', 'wenzhuman'],
+    'Anton Gulenko': ['anton gulenko', 'anton_gulenko'],
+    'Richard Lancaster': ['richardlancaster'],
+    'William Leslie': ['William ML Leslie'],
+    'Spenser Bauman': ['Spenser Andrew Bauman'],
+    'Raffael Tfirst': ['raffael.tfirst at gmail.com'],
+    'timo': ['timo at eistee.fritz.box'],
+    'Jasper Schulz': ['Jasper.Schulz', 'jbs'],
+    'Aaron Gallagher': ['"Aaron Gallagher'],
+    'Yasir Suhail': ['yasirs'],
     'Squeaky': ['squeaky'],
-    "Amaury Forgeot d'Arc": ['amauryfa at gmail.com'],
     "Dodan Mihai": ['mihai.dodan at gmail.com'],
+    'Wim Lavrijsen': ['wlav'],
+    'Toon Verwaest': ['toon', 'tverwaes'],
+    'Seo Sanghyeon': ['sanxiyn'],
+    'Leonardo Santagada': ['santagada'],
+    'Laurence Tratt': ['ltratt'],
+    'Pieter Zieschang': ['pzieschang', 'p_zieschang at yahoo.de'],
+    'John Witulski': ['witulski'],
     }
 
 alias_map = {}
@@ -103,7 +110,8 @@
         return set()
     ignore_words = ['around', 'consulting', 'yesterday', 'for a bit', 'thanks',
                     'in-progress', 'bits of', 'even a little', 'floating',
-                    'a bit', 'reviewing']
+                    'a bit', 'reviewing', 'looking', 'advising', 'partly', 'ish',
+                    'watching', 'mostly', 'jumping']
     sep_words = ['and', ';', '+', '/', 'with special  by']
     nicknames = match.group(1)
     for word in ignore_words:
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -7,9 +7,12 @@
 
 .. branch: cppyy-packaging
 
-Upgrade to backend 1.1.0, improved handling of templated methods and
+Upgrade to backend 1.2.0, improved handling of templated methods and
 functions (in particular automatic deduction of types), improved pythonization
-interface, and a range of compatibility fixes for Python3
+interface, range of compatibility fixes for Python3, free functions now take
+fast libffi path when possible, moves for strings (incl. from Python str),
+easier/faster handling of std::vector by numpy, improved and faster object
+identity preservation
 
 .. branch: socket_default_timeout_blockingness
 
diff --git a/pypy/module/_cppyy/converter.py b/pypy/module/_cppyy/converter.py
--- a/pypy/module/_cppyy/converter.py
+++ b/pypy/module/_cppyy/converter.py
@@ -1,15 +1,12 @@
 import sys
 
 from pypy.interpreter.error import OperationError, oefmt
-
 from rpython.rtyper.lltypesystem import rffi, lltype
 from rpython.rlib.rarithmetic import r_singlefloat, r_longfloat
 from rpython.rlib import rfloat, rawrefcount
-
 from pypy.module._rawffi.interp_rawffi import letter2tp
 from pypy.module._rawffi.array import W_ArrayInstance
-
-from pypy.module._cppyy import helper, capi, ffitypes
+from pypy.module._cppyy import helper, capi, ffitypes, lowlevelviews
 
 # Converter objects are used to translate between RPython and C++. They are
 # defined by the type name for which they provide conversion. Uses are for
@@ -149,7 +146,8 @@
         # read access, so no copy needed
         address_value = self._get_raw_address(space, w_obj, offset)
         address = rffi.cast(rffi.ULONG, address_value)
-        return W_ArrayInstance(space, letter2tp(space, self.typecode), self.size, address)
+        return lowlevelviews.W_LowLevelView(
+            space, letter2tp(space, self.typecode), self.size, address)
 
     def to_memory(self, space, w_obj, w_value, offset):
         # copy the full array (uses byte copy for now)
@@ -190,7 +188,8 @@
         # read access, so no copy needed
         address_value = self._get_raw_address(space, w_obj, offset)
         address = rffi.cast(rffi.ULONGP, address_value)
-        return W_ArrayInstance(space, letter2tp(space, self.typecode), self.size, address[0])
+        return lowlevelviews.W_LowLevelView(
+            space, letter2tp(space, self.typecode), self.size, address[0])
 
     def to_memory(self, space, w_obj, w_value, offset):
         # copy only the pointer value
@@ -438,7 +437,7 @@
             from pypy.module._cppyy import interp_cppyy
             return interp_cppyy.get_nullptr(space)
         shape = letter2tp(space, 'P')
-        return W_ArrayInstance(space, shape, sys.maxint/shape.size, ptrval)
+        return lowlevelviews.W_LowLevelView(space, shape, sys.maxint/shape.size, ptrval)
 
     def to_memory(self, space, w_obj, w_value, offset):
         address = rffi.cast(rffi.VOIDPP, self._get_raw_address(space, w_obj, offset))
@@ -491,7 +490,7 @@
         from pypy.module._cppyy.interp_cppyy import W_CPPInstance
         if isinstance(w_obj, W_CPPInstance):
             from pypy.module._cppyy.interp_cppyy import INSTANCE_FLAGS_IS_RVALUE
-            if w_obj.flags & INSTANCE_FLAGS_IS_RVALUE:
+            if w_obj.rt_flags & INSTANCE_FLAGS_IS_RVALUE:
                 # reject moves as all are explicit
                 raise ValueError("lvalue expected")
             if capi.c_is_subtype(space, w_obj.clsdecl, self.clsdecl):
@@ -523,14 +522,14 @@
         from pypy.module._cppyy.interp_cppyy import W_CPPInstance, INSTANCE_FLAGS_IS_RVALUE
         obj = space.interp_w(W_CPPInstance, w_obj)
         if obj:
-            if obj.flags & INSTANCE_FLAGS_IS_RVALUE:
-                obj.flags &= ~INSTANCE_FLAGS_IS_RVALUE
+            if obj.rt_flags & INSTANCE_FLAGS_IS_RVALUE:
+                obj.rt_flags &= ~INSTANCE_FLAGS_IS_RVALUE
                 try:
                     return InstanceRefConverter._unwrap_object(self, space, w_obj)
                 except Exception:
                     # TODO: if the method fails on some other converter, then the next
                     # overload can not be an rvalue anymore
-                    obj.flags |= INSTANCE_FLAGS_IS_RVALUE
+                    obj.rt_flags |= INSTANCE_FLAGS_IS_RVALUE
                     raise
         raise oefmt(space.w_ValueError, "object is not an rvalue")
 
@@ -566,10 +565,6 @@
         from pypy.module._cppyy import interp_cppyy
         return interp_cppyy.wrap_cppinstance(space, address, self.clsdecl, do_cast=False)
 
-    def to_memory(self, space, w_obj, w_value, offset):
-        address = rffi.cast(rffi.VOIDPP, self._get_raw_address(space, w_obj, offset))
-        address[0] = rffi.cast(rffi.VOIDP, self._unwrap_object(space, w_value))
-
 class InstancePtrPtrConverter(InstancePtrConverter):
     typecode = 'o'
 
@@ -597,6 +592,25 @@
         return interp_cppyy.wrap_cppinstance(
             space, address, self.clsdecl, do_cast=False, is_ref=True)
 
+    def to_memory(self, space, w_obj, w_value, offset):
+        # the actual data member is of object* type, but we receive a pointer to that
+        # data member in order to modify its value, so by convention, the internal type
+        # used is object**
+        address = rffi.cast(rffi.VOIDPP, self._get_raw_address(space, w_obj, offset))
+        from pypy.module._cppyy.interp_cppyy import W_CPPInstance
+        cppinstance = space.interp_w(W_CPPInstance, w_value, can_be_None=True)
+        if cppinstance:
+            rawobject = cppinstance.get_rawobject()
+            offset = capi.c_base_offset(space, cppinstance.clsdecl, self.clsdecl, rawobject, 1)
+            obj_address = capi.direct_ptradd(rawobject, offset)
+            address[0] = rffi.cast(rffi.VOIDP, obj_address);
+            # register the value for potential recycling
+            from pypy.module._cppyy.interp_cppyy import memory_regulator
+            memory_regulator.register(cppinstance)
+        else:
+            raise oefmt(space.w_TypeError,
+                "cannot pass %T instance as %s", w_value, self.clsdecl.name)
+
     def finalize_call(self, space, w_obj):
         if self.ref_buffer:
             set_rawobject(space, w_obj, self.ref_buffer[0])
@@ -607,7 +621,6 @@
             self.ref_buffer = lltype.nullptr(rffi.VOIDPP.TO)
 
 class StdStringConverter(InstanceConverter):
-
     def __init__(self, space, extra):
         from pypy.module._cppyy import interp_cppyy
         cppclass = interp_cppyy.scope_byname(space, capi.std_string_name)
@@ -633,6 +646,34 @@
     def free_argument(self, space, arg):
         capi.c_destruct(space, self.clsdecl, rffi.cast(capi.C_OBJECT, rffi.cast(rffi.VOIDPP, arg)[0]))
 
+class StdStringMoveConverter(StdStringConverter):
+    def _unwrap_object(self, space, w_obj):
+        # moving is same as by-ref, but have to check that move is allowed
+        moveit_reason = 3
+        from pypy.module._cppyy.interp_cppyy import W_CPPInstance, INSTANCE_FLAGS_IS_RVALUE
+        try:
+            obj = space.interp_w(W_CPPInstance, w_obj)
+            if obj and obj.rt_flags & INSTANCE_FLAGS_IS_RVALUE:
+                obj.rt_flags &= ~INSTANCE_FLAGS_IS_RVALUE
+                moveit_reason = 1
+            else:
+                moveit_reason = 0
+        except:
+            pass
+
+        if moveit_reason:
+            try:
+                return StdStringConverter._unwrap_object(self, space, w_obj)
+            except Exception:
+                 if moveit_reason == 1:
+                    # TODO: if the method fails on some other converter, then the next
+                    # overload can not be an rvalue anymore
+                    obj = space.interp_w(W_CPPInstance, w_obj)
+                    obj.rt_flags |= INSTANCE_FLAGS_IS_RVALUE
+                    raise
+
+        raise oefmt(space.w_ValueError, "object is not an rvalue")
+
 class StdStringRefConverter(InstancePtrConverter):
     _immutable_fields_ = ['cppclass', 'typecode']
     typecode    = 'V'
@@ -885,6 +926,7 @@
 _converters["std::basic_string<char>"]           = StdStringConverter
 _converters["const std::basic_string<char>&"]    = StdStringConverter     # TODO: shouldn't copy
 _converters["std::basic_string<char>&"]          = StdStringRefConverter
+_converters["std::basic_string<char>&&"]         = StdStringMoveConverter
 
 _converters["PyObject*"]                         = PyObjectConverter
 
@@ -1002,6 +1044,7 @@
         ("std::basic_string<char>",         "string"),
         ("const std::basic_string<char>&",  "const string&"),
         ("std::basic_string<char>&",        "string&"),
+        ("std::basic_string<char>&&",       "string&&"),
 
         ("PyObject*",                       "_object*"),
     )
diff --git a/pypy/module/_cppyy/executor.py b/pypy/module/_cppyy/executor.py
--- a/pypy/module/_cppyy/executor.py
+++ b/pypy/module/_cppyy/executor.py
@@ -1,14 +1,10 @@
 import sys
 
 from pypy.interpreter.error import oefmt
-
 from rpython.rtyper.lltypesystem import rffi, lltype
 from rpython.rlib import jit_libffi
-
 from pypy.module._rawffi.interp_rawffi import letter2tp
-from pypy.module._rawffi.array import W_Array, W_ArrayInstance
-
-from pypy.module._cppyy import helper, capi, ffitypes
+from pypy.module._cppyy import helper, capi, ffitypes, lowlevelviews
 
 # Executor objects are used to dispatch C++ methods. They are defined by their
 # return type only: arguments are converted by Converter objects, and Executors
@@ -60,7 +56,7 @@
             from pypy.module._cppyy import interp_cppyy
             return interp_cppyy.get_nullptr(space)
         shape = letter2tp(space, self.typecode)
-        return W_ArrayInstance(space, shape, sys.maxint/shape.size, ptrval)
+        return lowlevelviews.W_LowLevelView(space, shape, sys.maxint/shape.size, ptrval)
 
 
 class VoidExecutor(Executor):
@@ -98,10 +94,10 @@
     def __init__(self, space, extra):
         Executor.__init__(self, space, extra)
         self.do_assign = False
-        self.item = rffi.cast(self.c_type, 0)
+        self.w_item = space.w_None
 
     def set_item(self, space, w_item):
-        self.item = self._unwrap_object(space, w_item)
+        self.w_item = w_item
         self.do_assign = True
 
     #def _wrap_object(self, space, obj):
@@ -109,7 +105,7 @@
 
     def _wrap_reference(self, space, rffiptr):
         if self.do_assign:
-            rffiptr[0] = self.item
+            rffiptr[0] = rffi.cast(self.c_type, self._unwrap_object(space, self.w_item))
         self.do_assign = False
         return self._wrap_object(space, rffiptr[0])    # all paths, for rtyper
 
diff --git a/pypy/module/_cppyy/interp_cppyy.py b/pypy/module/_cppyy/interp_cppyy.py
--- a/pypy/module/_cppyy/interp_cppyy.py
+++ b/pypy/module/_cppyy/interp_cppyy.py
@@ -41,6 +41,7 @@
              'void**'        : 100,
              'float'         :  30,
              'double'        :  10,
+             'bool'          :   1,
              'const string&' :   1, } # solves a specific string ctor overload
 
 from rpython.rlib.listsort import make_timsort_class
@@ -167,6 +168,7 @@
 #
 #  W_CPPOverload:                 instance methods (base class)
 #  W_CPPConstructorOverload:      constructors
+#  W_CPPAbstractCtorOverload:     to provent instantiation of abstract classes
 #  W_CPPStaticOverload:           free and static functions
 #  W_CPPTemplateOverload:         templated methods
 #  W_CPPTemplateStaticOverload:   templated free and static functions
@@ -258,7 +260,8 @@
         jit.promote(self)
         cif_descr = self.cif_descr
         # add extra space for const-ref support (see converter.py)
-        buffer = lltype.malloc(rffi.CCHARP.TO, cif_descr.exchange_size+len(self.arg_defs)*rffi.sizeof(rffi.DOUBLE), flavor='raw')
+        buffer = lltype.malloc(rffi.CCHARP.TO,
+            cif_descr.exchange_size+len(self.arg_defs)*rffi.sizeof(rffi.DOUBLE), flavor='raw')
         thisoff = 0
         try:
             if cppthis:
@@ -412,8 +415,10 @@
 
     def priority(self):
         total_arg_priority = 0
-        for p in [priority.get(arg_type, 0) for arg_type, arg_dflt in self.arg_defs]:
-            total_arg_priority += p
+        for arg_type, arg_dflt in self.arg_defs:
+            total_arg_priority += priority.get(arg_type, 0)
+            if '&&' in arg_type:
+                total_arg_priority += 100
         return total_arg_priority
 
     @rgc.must_be_light_finalizer
@@ -433,7 +438,7 @@
 
 class CPPSetItem(CPPMethod):
     """Method dispatcher specific to Python's __setitem__ mapped onto C++'s
-    operator[](int). The former function takes an extra argument to assign to
+    operator[](T). The former function takes an extra argument to assign to
     the return type of the latter."""
 
     _attrs_ = []
@@ -505,14 +510,17 @@
     def descr_get(self, w_obj, w_cls=None):
         """functionobject.__get__(obj[, type]) -> method"""
         # TODO: check validity of w_cls if given
+        # TODO: this does not work for Python 3, which does not have
+        #  unbound methods (probably no common code possible, see also
+        #  pypy/interpreter/function.py)
         space = self.space
         asking_for_bound = (space.is_none(w_cls) or
                             not space.is_w(w_obj, space.w_None) or
                             space.is_w(w_cls, space.type(space.w_None)))
         if asking_for_bound:
-            return MethodWithProps(space, self, w_obj)
+            return MethodWithProps(space, self, w_obj, w_cls)
         else:
-            return self  # unbound methods don't exist in Python 3, return self
+            return MethodWithProps(space, self, None, w_cls)
 
     @unwrap_spec(args_w='args_w')
     def call_args(self, args_w):
@@ -584,6 +592,16 @@
             sig += '\n'+self.functions[i].prototype()
         return self.space.newtext(sig)
 
+    @unwrap_spec(signature='text')
+    def mp_overload(self, signature):
+        sig = '(%s)' % signature
+        for f in self.functions:
+            if f.signature(False) == sig:
+                if isinstance(self, W_CPPStaticOverload):
+                    return W_CPPStaticOverload(self.space, self.scope, [f])
+                return W_CPPOverload(self.space, self.scope, [f])
+        raise oefmt(self.space.w_LookupError, "signature '%s' not found", signature)
+
     # allow user to determine ffi use rules per overload
     def fget_useffi(self, space):
         return space.newbool(bool(self.flags & OVERLOAD_FLAGS_USE_FFI))
@@ -607,10 +625,11 @@
 
 W_CPPOverload.typedef = TypeDef(
     'CPPOverload',
-    __get__    = interp2app(W_CPPOverload.descr_get),
-    __call__   = interp2app(W_CPPOverload.call_args),
-    __useffi__ = GetSetProperty(W_CPPOverload.fget_useffi, W_CPPOverload.fset_useffi),
-    __doc__    = GetSetProperty(W_CPPOverload.fget_doc)
+    __get__      = interp2app(W_CPPOverload.descr_get),
+    __call__     = interp2app(W_CPPOverload.call_args),
+    __useffi__   = GetSetProperty(W_CPPOverload.fget_useffi, W_CPPOverload.fset_useffi),
+    __overload__ = interp2app(W_CPPOverload.mp_overload),
+    __doc__      = GetSetProperty(W_CPPOverload.fget_doc)
 )
 
 
@@ -626,27 +645,24 @@
             # onto a class and w_this should be set
             cppinstance = self.space.interp_w(W_CPPInstance, w_obj)
             if cppinstance.clsdecl.handle != self.scope.handle:
-                return MethodWithProps(self.space, self, w_obj)    # bound
+                return MethodWithProps(self.space, self, w_obj, w_cls)    # bound
         return self      # unbound
 
     @unwrap_spec(args_w='args_w')
     def call_args(self, args_w):
         jit.promote(self)
-        #if isinstance(args_w[0], W_CPPInstance):
-            # free function used as bound method, leave in place
         return self.call_impl(capi.C_NULL_OBJECT, args_w)
-        # free functions are implemented as methods of 'namespace' classes, remove 'instance'
-        #return self.call_impl(capi.C_NULL_OBJECT, args_w[1:])
 
     def __repr__(self):
         return "W_CPPStaticOverload(%s)" % [f.prototype() for f in self.functions]
 
 W_CPPStaticOverload.typedef = TypeDef(
     'CPPStaticOverload',
-    __get__    = interp2app(W_CPPStaticOverload.descr_get),
-    __call__   = interp2app(W_CPPStaticOverload.call_args),
-    __useffi__ = GetSetProperty(W_CPPStaticOverload.fget_useffi, W_CPPStaticOverload.fset_useffi),
-    __doc__    = GetSetProperty(W_CPPStaticOverload.fget_doc)
+    __get__      = interp2app(W_CPPStaticOverload.descr_get),
+    __call__     = interp2app(W_CPPStaticOverload.call_args),
+    __useffi__   = GetSetProperty(W_CPPStaticOverload.fget_useffi, W_CPPStaticOverload.fset_useffi),
+    __overload__ = interp2app(W_CPPStaticOverload.mp_overload),
+    __doc__      = GetSetProperty(W_CPPStaticOverload.fget_doc)
 )
 
 
@@ -660,11 +676,6 @@
     @unwrap_spec(args_w='args_w')
     def call_args(self, args_w):
         jit.promote(self)
-        # TODO: factor out the following:
-        if capi.c_is_abstract(self.space, self.scope.handle):
-            raise oefmt(self.space.w_TypeError,
-                        "cannot instantiate abstract class '%s'",
-                        self.scope.name)
         cppinstance = self.space.interp_w(W_CPPInstance, args_w[0])
         w_result = self.call_impl(rffi.cast(capi.C_OBJECT, self.scope.handle), args_w[1:])
         newthis = rffi.cast(capi.C_OBJECT, self.space.uint_w(w_result))
@@ -677,9 +688,27 @@
 
 W_CPPConstructorOverload.typedef = TypeDef(
     'CPPConstructorOverload',
-    __get__    = interp2app(W_CPPConstructorOverload.descr_get),
-    __call__   = interp2app(W_CPPConstructorOverload.call_args),
-    __doc__    = GetSetProperty(W_CPPConstructorOverload.fget_doc)
+    __get__      = interp2app(W_CPPConstructorOverload.descr_get),
+    __call__     = interp2app(W_CPPConstructorOverload.call_args),
+    __overload__ = interp2app(W_CPPConstructorOverload.mp_overload),
+    __doc__      = GetSetProperty(W_CPPConstructorOverload.fget_doc)
+)
+
+class W_CPPAbstractCtorOverload(W_CPPOverload):
+    _attrs_ = []
+
+    @unwrap_spec(args_w='args_w')
+    def call_args(self, args_w):
+        raise oefmt(self.space.w_TypeError,
+            "cannot instantiate abstract class '%s'", self.scope.name)
+
+    def __repr__(self):
+        return "W_CPPAbstractCtorOverload"
+
+W_CPPAbstractCtorOverload.typedef = TypeDef(
+    'CPPAbstractCtorOverload',
+    __get__    = interp2app(W_CPPAbstractCtorOverload.descr_get),
+    __call__   = interp2app(W_CPPAbstractCtorOverload.call_args),
 )
 
 
@@ -702,7 +731,7 @@
                     if args_w:
                         # try to specialize the type match for the given object
                         cppinstance = self.space.interp_w(W_CPPInstance, args_w[i])
-                        if cppinstance.flags & INSTANCE_FLAGS_IS_RVALUE:
+                        if cppinstance.rt_flags & INSTANCE_FLAGS_IS_RVALUE:
                             sugar = "&&"
                         elif cppinstance.flags & INSTANCE_FLAGS_IS_REF:
                             sugar = "*"
@@ -842,11 +871,11 @@
 
 W_CPPTemplateOverload.typedef = TypeDef(
     'CPPTemplateOverload',
-    __get__     = interp2app(W_CPPTemplateOverload.descr_get),
-    __getitem__ = interp2app(W_CPPTemplateOverload.getitem),
-    __call__    = interp2app(W_CPPTemplateOverload.call_args),
-    __useffi__  = GetSetProperty(W_CPPTemplateOverload.fget_useffi, W_CPPTemplateOverload.fset_useffi),
-    __doc__     = GetSetProperty(W_CPPTemplateOverload.fget_doc)
+    __get__      = interp2app(W_CPPTemplateOverload.descr_get),
+    __getitem__  = interp2app(W_CPPTemplateOverload.getitem),
+    __call__     = interp2app(W_CPPTemplateOverload.call_args),
+    __useffi__   = GetSetProperty(W_CPPTemplateOverload.fget_useffi, W_CPPTemplateOverload.fset_useffi),
+    __doc__      = GetSetProperty(W_CPPTemplateOverload.fget_doc)
 )
 
 class W_CPPTemplateStaticOverload(W_CPPStaticOverload, TemplateOverloadMixin):
@@ -900,11 +929,11 @@
 
 W_CPPTemplateStaticOverload.typedef = TypeDef(
     'CPPTemplateStaticOverload',
-    __get__     = interp2app(W_CPPTemplateStaticOverload.descr_get),
-    __getitem__ = interp2app(W_CPPTemplateStaticOverload.getitem),
-    __call__    = interp2app(W_CPPTemplateStaticOverload.call_args),
-    __useffi__  = GetSetProperty(W_CPPTemplateStaticOverload.fget_useffi, W_CPPTemplateStaticOverload.fset_useffi),
-    __doc__     = GetSetProperty(W_CPPTemplateStaticOverload.fget_doc)
+    __get__      = interp2app(W_CPPTemplateStaticOverload.descr_get),
+    __getitem__  = interp2app(W_CPPTemplateStaticOverload.getitem),
+    __call__     = interp2app(W_CPPTemplateStaticOverload.call_args),
+    __useffi__   = GetSetProperty(W_CPPTemplateStaticOverload.fget_useffi, W_CPPTemplateStaticOverload.fset_useffi),
+    __doc__      = GetSetProperty(W_CPPTemplateStaticOverload.fget_doc)
 )
 
 
@@ -932,6 +961,7 @@
 
     def _get_offset(self, cppinstance):
         if cppinstance:
+            assert isinstance(cppinstance.clsdecl, W_CPPClassDecl)
             assert lltype.typeOf(cppinstance.clsdecl.handle) == lltype.typeOf(self.scope.handle)
             offset = self.offset + cppinstance.clsdecl.get_base_offset(cppinstance, self.scope)
         else:
@@ -1077,6 +1107,8 @@
         sig = '(%s)' % signature
         for f in overload.functions:
             if f.signature(False) == sig:
+                if isinstance(overload, W_CPPStaticOverload):
+                    return W_CPPStaticOverload(self.space, self, [f])
                 return W_CPPOverload(self.space, self, [f])
         raise oefmt(self.space.w_LookupError, "no overload matches signature")
 
@@ -1172,9 +1204,13 @@
 
 
 class W_CPPClassDecl(W_CPPScopeDecl):
-    _attrs_ = ['space', 'handle', 'name', 'overloads', 'datamembers']
+    _attrs_ = ['space', 'handle', 'name', 'overloads', 'datamembers', 'cppobjects']
     _immutable_fields_ = ['handle', 'name', 'overloads[*]', 'datamembers[*]']
 
+    def __init__(self, space, opaque_handle, final_scoped_name):
+        W_CPPScopeDecl.__init__(self, space, opaque_handle, final_scoped_name)
+        self.cppobjects = rweakref.RWeakValueDictionary(int, W_CPPInstance)
+
     def _build_overloads(self):
         assert len(self.overloads) == 0
         methods_tmp = {}; ftype_tmp = {}
@@ -1213,7 +1249,10 @@
             ftype = ftype_tmp[pyname]
             CPPMethodSort(methods).sort()
             if ftype & FUNCTION_IS_CONSTRUCTOR:
-                overload = W_CPPConstructorOverload(self.space, self, methods[:])
+                if capi.c_is_abstract(self.space, self.handle):
+                    overload = W_CPPAbstractCtorOverload(self.space, self, methods[:])
+                else:
+                    overload = W_CPPConstructorOverload(self.space, self, methods[:])
             elif ftype & FUNCTION_IS_STATIC:
                 if ftype & FUNCTION_IS_TEMPLATE:
                     cppname = capi.c_method_name(self.space, methods[0].cppmethod)
@@ -1279,10 +1318,12 @@
         raise self.missing_attribute_error(name)
 
     def get_base_offset(self, cppinstance, calling_scope):
+        assert isinstance(cppinstance.clsdecl, W_CPPClassDecl)
         assert self == cppinstance.clsdecl
         return 0
 
     def get_cppthis(self, cppinstance, calling_scope):
+        assert isinstance(cppinstance.clsdecl, W_CPPClassDecl)
         assert self == cppinstance.clsdecl
         return cppinstance.get_rawobject()
 
@@ -1318,12 +1359,14 @@
 
 class W_CPPComplexClassDecl(W_CPPClassDecl):
     def get_base_offset(self, cppinstance, calling_scope):
+        assert isinstance(cppinstance.clsdecl, W_CPPComplexClassDecl)
         assert self == cppinstance.clsdecl
         offset = capi.c_base_offset(self.space,
-                                    self, calling_scope, cppinstance.get_rawobject(), 1)
+            self, calling_scope, cppinstance.get_rawobject(), 1)
         return offset
 
     def get_cppthis(self, cppinstance, calling_scope):
+        assert isinstance(cppinstance.clsdecl, W_CPPComplexClassDecl)
         assert self == cppinstance.clsdecl
         offset = self.get_base_offset(cppinstance, calling_scope)
         return capi.direct_ptradd(cppinstance.get_rawobject(), offset)
@@ -1343,9 +1386,9 @@
 
 
 class W_CPPInstance(W_Root):
-    _attrs_ = ['space', 'clsdecl', '_rawobject', 'smartdecl', 'deref', 'flags',
+    _attrs_ = ['space', 'clsdecl', '_rawobject', 'smartdecl', 'deref', 'flags', 'rt_flags',
                'finalizer_registered']
-    _immutable_fields_ = ['clsdecl', 'smartdecl', 'deref']
+    _immutable_fields_ = ['clsdecl', 'smartdecl', 'deref', 'flags']
 
     finalizer_registered = False
 
@@ -1353,6 +1396,7 @@
                  smartdecl=None, deref=rffi.cast(capi.C_METHOD, 0)):
         self.space = space
         self.clsdecl = decl
+        assert isinstance(self.clsdecl, W_CPPClassDecl)
         assert lltype.typeOf(rawobject) == capi.C_OBJECT
         assert not isref or rawobject
         self._rawobject = rawobject
@@ -1360,15 +1404,16 @@
         self.flags = 0
         if isref or (smartdecl and deref):
             self.flags |= INSTANCE_FLAGS_IS_REF
+        self.rt_flags = 0
         if python_owns:
-            self.flags |= INSTANCE_FLAGS_PYTHON_OWNS
+            self.rt_flags |= INSTANCE_FLAGS_PYTHON_OWNS
             self._opt_register_finalizer()
         self.smartdecl = smartdecl
         self.deref     = deref
 
     def _opt_register_finalizer(self):
         if not self.finalizer_registered and not hasattr(self.space, "fake"):
-            assert self.flags & INSTANCE_FLAGS_PYTHON_OWNS
+            assert self.rt_flags & INSTANCE_FLAGS_PYTHON_OWNS
             self.register_finalizer(self.space)
             self.finalizer_registered = True
 
@@ -1380,17 +1425,18 @@
 
     # allow user to determine ownership rules on a per object level
     def fget_python_owns(self, space):
-        return space.newbool(bool(self.flags & INSTANCE_FLAGS_PYTHON_OWNS))
+        return space.newbool(bool(self.rt_flags & INSTANCE_FLAGS_PYTHON_OWNS))
 
     @unwrap_spec(value=bool)
     def fset_python_owns(self, space, value):
         if space.is_true(value):
-            self.flags |= INSTANCE_FLAGS_PYTHON_OWNS
+            self.rt_flags |= INSTANCE_FLAGS_PYTHON_OWNS
             self._opt_register_finalizer()
         else:
-            self.flags &= ~INSTANCE_FLAGS_PYTHON_OWNS
+            self.rt_flags &= ~INSTANCE_FLAGS_PYTHON_OWNS
 
     def get_cppthis(self, calling_scope):
+        assert isinstance(self.clsdecl, W_CPPClassDecl)
         return self.clsdecl.get_cppthis(self, calling_scope)
 
     def get_rawobject(self):
@@ -1497,6 +1543,7 @@
 
     def destruct(self):
         if self._rawobject:
+            assert isinstance(self.clsdecl, W_CPPClassDecl)
             if self.smartdecl and self.deref:
                 klass = self.smartdecl
             elif not (self.flags & INSTANCE_FLAGS_IS_REF):
@@ -1508,7 +1555,7 @@
             self._rawobject = capi.C_NULL_OBJECT
 
     def _finalize_(self):
-        if self.flags & INSTANCE_FLAGS_PYTHON_OWNS:
+        if self.rt_flags & INSTANCE_FLAGS_PYTHON_OWNS:
             self.destruct()
 
 W_CPPInstance.typedef = TypeDef(
@@ -1530,31 +1577,33 @@
 
 
 class MemoryRegulator:
-    # TODO: (?) An object address is not unique if e.g. the class has a
-    # public data member of class type at the start of its definition and
-    # has no virtual functions. A _key class that hashes on address and
-    # type would be better, but my attempt failed in the rtyper, claiming
-    # a call on None ("None()") and needed a default ctor. (??)
-    # Note that for now, the associated test carries an m_padding to make
-    # a difference in the addresses.
-    def __init__(self):
-        self.objects = rweakref.RWeakValueDictionary(int, W_CPPInstance)
+    _immutable_ = True
 
-    def register(self, obj):
+    @staticmethod
+    def register(obj):
         if not obj._rawobject:
             return
-        int_address = int(rffi.cast(rffi.LONG, obj._rawobject))
-        self.objects.set(int_address, obj)
+        addr_as_int = int(rffi.cast(rffi.LONG, obj.get_rawobject()))
+        clsdecl = obj.clsdecl
+        assert isinstance(clsdecl, W_CPPClassDecl)
+        clsdecl.cppobjects.set(addr_as_int, obj)
 
-    def unregister(self, obj):
+    @staticmethod
+    def unregister(obj):
         if not obj._rawobject:
             return
-        int_address = int(rffi.cast(rffi.LONG, obj._rawobject))
-        self.objects.set(int_address, None)
+        addr_as_int = int(rffi.cast(rffi.LONG, obj.get_rawobject()))
+        clsdecl = obj.clsdecl
+        assert isinstance(clsdecl, W_CPPClassDecl)
+        clsdecl.cppobjects.set(addr_as_int, None) # actually deletes (pops)
 
-    def retrieve(self, address):
-        int_address = int(rffi.cast(rffi.LONG, address))
-        return self.objects.get(int_address)
+    @staticmethod
+    def retrieve(clsdecl, address):
+        if not address:
+            return None
+        addr_as_int = int(rffi.cast(rffi.LONG, address))
+        assert isinstance(clsdecl, W_CPPClassDecl)
+        return clsdecl.cppobjects.get(addr_as_int)
 
 memory_regulator = MemoryRegulator()
 
@@ -1600,8 +1649,11 @@
 
     # try to recycle existing object if this one is not newly created
     if not fresh and rawobject:
-        obj = memory_regulator.retrieve(rawobject)
-        if obj is not None and obj.clsdecl is clsdecl:
+        address = rawobject
+        if is_ref:
+            address = rffi.cast(capi.C_OBJECT, rffi.cast(rffi.VOIDPP, address)[0])
+        obj = memory_regulator.retrieve(clsdecl, address)
+        if obj is not None:
             return obj
 
     # fresh creation
@@ -1652,7 +1704,7 @@
     """Casts the given instance into an C++-style rvalue."""
     obj = space.interp_w(W_CPPInstance, w_obj)
     if obj:
-        obj.flags |= INSTANCE_FLAGS_IS_RVALUE
+        obj.rt_flags |= INSTANCE_FLAGS_IS_RVALUE
     return w_obj
 
 
diff --git a/pypy/module/_cppyy/lowlevelviews.py b/pypy/module/_cppyy/lowlevelviews.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/_cppyy/lowlevelviews.py
@@ -0,0 +1,54 @@
+# Naked C++ pointers carry no useful size or layout information, but often
+# such information is externnally available. Low level views are arrays with
+# a few more methods allowing such information to be set. Afterwards, it is
+# simple to pass these views on to e.g. numpy (w/o the need to copy).
+
+from pypy.interpreter.error import oefmt
+from pypy.interpreter.gateway import interp2app, unwrap_spec
+from pypy.interpreter.typedef import TypeDef, GetSetProperty, interp_attrproperty_w
+from pypy.module._rawffi.array import W_ArrayInstance
+
+
+class W_LowLevelView(W_ArrayInstance):
+    def __init__(self, space, shape, length, address):
+        assert address   # if not address, base class will allocate memory
+        W_ArrayInstance.__init__(self, space, shape, length, address)
+
+    @unwrap_spec(args_w='args_w')
+    def reshape(self, space, args_w):
+        # llviews are only created from non-zero addresses, so we only need
+        # to adjust length and shape
+
+        nargs = len(args_w)
+        if nargs == 0:
+            raise oefmt(space.w_TypeError, "reshape expects a tuple argument")
+
+        newshape_w = args_w
+        if nargs != 1 or not space.isinstance_w(args_w[0], space.w_tuple) or \
+               not space.len_w(args_w[0]) == 1:
+            raise oefmt(space.w_TypeError,
+                "tuple object of length 1 expected, received %T", args_w[0])
+
+        w_shape = args_w[0]
+
+        # shape in W_ArrayInstance-speak is somewhat different from what
+        # e.g. numpy thinks of it: self.shape contains the info (itemcode,
+        # size, etc.) of a single entry; length is user-facing shape
+        self.length = space.int_w(space.getitem(w_shape, space.newint(0)))
+
+
+W_LowLevelView.typedef = TypeDef(
+    'LowLevelView',
+    __repr__    = interp2app(W_LowLevelView.descr_repr),
+    __setitem__ = interp2app(W_LowLevelView.descr_setitem),
+    __getitem__ = interp2app(W_LowLevelView.descr_getitem),
+    __len__     = interp2app(W_LowLevelView.getlength),
+    buffer      = GetSetProperty(W_LowLevelView.getbuffer),
+    shape       = interp_attrproperty_w('shape', W_LowLevelView),
+    free        = interp2app(W_LowLevelView.free),
+    byptr       = interp2app(W_LowLevelView.byptr),
+    itemaddress = interp2app(W_LowLevelView.descr_itemaddress),
+    reshape     = interp2app(W_LowLevelView.reshape),
+)
+W_ArrayInstance.typedef.acceptable_as_base_class = False
+
diff --git a/pypy/module/_cppyy/pythonify.py b/pypy/module/_cppyy/pythonify.py
--- a/pypy/module/_cppyy/pythonify.py
+++ b/pypy/module/_cppyy/pythonify.py
@@ -73,7 +73,8 @@
 
 # C++ namespace base class (the C++ class base class defined in _post_import_startup)
 class CPPNamespace(with_metaclass(CPPNamespaceMeta, object)):
-     pass
+    def __init__(self):
+        raise TypeError("cannot instantiate namespace '%s'", self.__cppname__)
 
 
 # TODO: this can be moved to the interp level (and share template argument
@@ -450,6 +451,14 @@
             return self.__real_init__(*args)
         pyclass.__init__ = vector_init
 
+        # size-up the return of data()
+        pyclass.__real_data = pyclass.data
+        def data_with_len(self):
+            arr = self.__real_data()
+            arr.reshape((len(self),))
+            return arr
+        pyclass.data = data_with_len
+
     # combine __getitem__ and __len__ to make a pythonized __getitem__
     if '__getitem__' in pyclass.__dict__ and '__len__' in pyclass.__dict__:
         pyclass._getitem__unchecked = pyclass.__getitem__
diff --git a/pypy/module/_cppyy/test/advancedcpp.cxx b/pypy/module/_cppyy/test/advancedcpp.cxx
--- a/pypy/module/_cppyy/test/advancedcpp.cxx
+++ b/pypy/module/_cppyy/test/advancedcpp.cxx
@@ -4,20 +4,32 @@
 
 
 // for testing of default arguments
-#define IMPLEMENT_DEFAULTER_CLASS(type, tname)                               \
+#define IMPLEMENT_DEFAULTERS(type, tname)                                    \
 tname##_defaulter::tname##_defaulter(type a, type b, type c) {               \
-   m_a = a; m_b = b; m_c = c;                                                \
+    m_a = a; m_b = b; m_c = c;                                               \
+}                                                                            \
+type tname##_defaulter_func(int idx, type a, type b, type c) {               \
+    if (idx == 0) return a;                                                  \
+    if (idx == 1) return b;                                                  \
+    if (idx == 2) return c;                                                  \
+    return (type)idx;                                                        \
 }
-IMPLEMENT_DEFAULTER_CLASS(short, short)
-IMPLEMENT_DEFAULTER_CLASS(unsigned short, ushort)
-IMPLEMENT_DEFAULTER_CLASS(int, int)
-IMPLEMENT_DEFAULTER_CLASS(unsigned, uint)
-IMPLEMENT_DEFAULTER_CLASS(long, long)
-IMPLEMENT_DEFAULTER_CLASS(unsigned long, ulong)
-IMPLEMENT_DEFAULTER_CLASS(long long, llong)
-IMPLEMENT_DEFAULTER_CLASS(unsigned long long, ullong)
-IMPLEMENT_DEFAULTER_CLASS(float, float)
-IMPLEMENT_DEFAULTER_CLASS(double, double)
+IMPLEMENT_DEFAULTERS(short, short)
+IMPLEMENT_DEFAULTERS(unsigned short, ushort)
+IMPLEMENT_DEFAULTERS(int, int)
+IMPLEMENT_DEFAULTERS(unsigned, uint)
+IMPLEMENT_DEFAULTERS(long, long)
+IMPLEMENT_DEFAULTERS(unsigned long, ulong)
+IMPLEMENT_DEFAULTERS(long long, llong)
+IMPLEMENT_DEFAULTERS(unsigned long long, ullong)
+IMPLEMENT_DEFAULTERS(float, float)
+IMPLEMENT_DEFAULTERS(double, double)
+
+std::string string_defaulter_func(int idx, const std::string& name1, std::string name2) {
+    if (idx == 0) return name1;
+    if (idx == 1) return name2;
+    return "mies";
+}
 
 
 // for esoteric inheritance testing
@@ -77,11 +89,11 @@
 double my_global_array[500];
 static double sd = 1234.;
 double* my_global_ptr = &sd;
+const char my_global_string2[] = "zus jet teun";
 some_int_holder my_global_int_holders[5] = {
     some_int_holder(13), some_int_holder(42), some_int_holder(88),
     some_int_holder(-1), some_int_holder(17) };
 
-
 // for life-line and identity testing
 int some_class_with_data::some_data::s_num_data = 0;
 
diff --git a/pypy/module/_cppyy/test/advancedcpp.h b/pypy/module/_cppyy/test/advancedcpp.h
--- a/pypy/module/_cppyy/test/advancedcpp.h
+++ b/pypy/module/_cppyy/test/advancedcpp.h
@@ -4,24 +4,27 @@
 
 
 //===========================================================================
-#define DECLARE_DEFAULTER_CLASS(type, tname)                                \
+#define DECLARE_DEFAULTERS(type, tname)                                     \
 class tname##_defaulter {                                                   \
 public:                                                                     \
     tname##_defaulter(type a = 11, type b = 22, type c = 33);               \
                                                                             \
 public:                                                                     \
     type m_a, m_b, m_c;                                                     \
-};
-DECLARE_DEFAULTER_CLASS(short, short)   // for testing of default arguments
-DECLARE_DEFAULTER_CLASS(unsigned short, ushort)
-DECLARE_DEFAULTER_CLASS(int, int)
-DECLARE_DEFAULTER_CLASS(unsigned, uint)
-DECLARE_DEFAULTER_CLASS(long, long)
-DECLARE_DEFAULTER_CLASS(unsigned long, ulong)
-DECLARE_DEFAULTER_CLASS(long long, llong)
-DECLARE_DEFAULTER_CLASS(unsigned long long, ullong)
-DECLARE_DEFAULTER_CLASS(float, float)
-DECLARE_DEFAULTER_CLASS(double, double)
+};                                                                          \
+type tname##_defaulter_func(int idx = 0, type a = 11, type b = 22, type c = 33);
+DECLARE_DEFAULTERS(short, short)   // for testing of default arguments
+DECLARE_DEFAULTERS(unsigned short, ushort)
+DECLARE_DEFAULTERS(int, int)
+DECLARE_DEFAULTERS(unsigned, uint)
+DECLARE_DEFAULTERS(long, long)
+DECLARE_DEFAULTERS(unsigned long, ulong)
+DECLARE_DEFAULTERS(long long, llong)
+DECLARE_DEFAULTERS(unsigned long long, ullong)
+DECLARE_DEFAULTERS(float, float)
+DECLARE_DEFAULTERS(double, double)
+
+std::string string_defaulter_func(int idx, const std::string& name1 = "aap", std::string name2 = "noot");
 
 
 //===========================================================================
@@ -274,7 +277,8 @@
 extern double my_global_double;    // a couple of globals for access testing
 extern double my_global_array[500];
 extern double* my_global_ptr;
-static const char my_global_string[] = "aap " " noot " " mies";
+static const char my_global_string1[] = "aap " " noot " " mies";
+extern const char my_global_string2[];
 
 class some_int_holder {
 public:
@@ -311,6 +315,11 @@
     some_data m_data;
 };
 
+class refers_to_self {             // for data member reuse testing
+public:
+    refers_to_self* m_other = nullptr;
+};
+
 
 //===========================================================================
 class pointer_pass {               // for testing passing of void*'s
@@ -419,3 +428,34 @@
     void throw_anything();
     void throw_exception();
 };
+
+
+//===========================================================================
+class UsingBase {                  // using declaration testing
+public:
+    UsingBase(int n = 13) : m_int(n) {} 
+    virtual char vcheck() { return 'A'; }
+    int m_int;
+};
+
+class UsingDerived : public UsingBase {
+public:
+    using UsingBase::UsingBase;
+    virtual char vcheck() { return 'B'; }
+    int m_int2 = 42;
+};
+
+
+//===========================================================================
+class TypedefToPrivateClass {      // typedef resolution testing
+private:
+    class PC {
+    public:
+        PC(int i) : m_val(i) {}
+        int m_val;
+    };
+
+public:
+    typedef PC PP;
+    PP f() { return PC(42); }
+};
diff --git a/pypy/module/_cppyy/test/advancedcpp.xml b/pypy/module/_cppyy/test/advancedcpp.xml
--- a/pypy/module/_cppyy/test/advancedcpp.xml
+++ b/pypy/module/_cppyy/test/advancedcpp.xml
@@ -1,6 +1,7 @@
 <lcgdict>
 
   <class pattern="*_defaulter" />
+  <function pattern="*_defaulter_func" />
 
   <class name="base_class" />
   <class name="derived_class" />
@@ -32,6 +33,7 @@
   <class name="some_convertible" />
   <class name="some_class_with_data" />
   <class name="some_class_with_data::some_data" />
+  <class name="refers_to_self" />
 
   <class name="some_comparable" />
   <function name="operator==" />
@@ -59,5 +61,7 @@
   <class name="overload_the_other_way" />
 
   <class name="Thrower" />
+  <class pattern="Using*" />
+  <class name="TypedefToPrivateClass" />
 
 </lcgdict>
diff --git a/pypy/module/_cppyy/test/stltypes.cxx b/pypy/module/_cppyy/test/stltypes.cxx
--- a/pypy/module/_cppyy/test/stltypes.cxx
+++ b/pypy/module/_cppyy/test/stltypes.cxx
@@ -1,6 +1,20 @@
 #include "stltypes.h"
 
 
+//- explicit instantiations of used comparisons
+#if defined __clang__
+namespace std {
+#define ns_prefix std::
+#elif defined(__GNUC__) || defined(__GNUG__)
+namespace __gnu_cxx {
+#define ns_prefix
+#endif
+template bool ns_prefix operator==(const std::vector<int>::iterator&,
+                         const std::vector<int>::iterator&);
+template bool ns_prefix operator!=(const std::vector<int>::iterator&,
+                         const std::vector<int>::iterator&);
+}
+
 //- class with lots of std::string handling
 stringy_class::stringy_class(const char* s) : m_string(s) {}
 
@@ -9,3 +23,33 @@
 
 void stringy_class::set_string1(const std::string& s) { m_string = s; }
 void stringy_class::set_string2(std::string s) { m_string = s; }
+
+
+//- helpers for testing array
+int ArrayTest::get_pp_px(Point** p, int idx) {
+    return p[idx]->px;
+}
+
+int ArrayTest::get_pp_py(Point** p, int idx) {
+    return p[idx]->py;
+}
+
+int ArrayTest::get_pa_px(Point* p[], int idx) {
+    return p[idx]->px;
+}
+
+int ArrayTest::get_pa_py(Point* p[], int idx) {
+    return p[idx]->py;
+}
+
+
+// helpers for string testing
+std::string str_array_1[3] = {"a", "b", "c"};
+std::string str_array_2[]  = {"d", "e", "f", "g"};
+std::string str_array_3[3][2] = {{"a", "b"}, {"c", "d"}, {"e", "f"}};
+std::string str_array_4[4][2][2] = {
+     {{"a", "b"}, {"c", "d"}},
+     {{"e", "f"}, {"g", "h"}},
+     {{"i", "j"}, {"k", "l"}},
+     {{"m", "n"}, {"o", "p"}},
+};
diff --git a/pypy/module/_cppyy/test/stltypes.h b/pypy/module/_cppyy/test/stltypes.h
--- a/pypy/module/_cppyy/test/stltypes.h
+++ b/pypy/module/_cppyy/test/stltypes.h
@@ -37,3 +37,49 @@
    std::string operator[](double) { return "double"; }
    std::string operator[](const std::string&) { return "string"; }
 };      
+
+
+//- instantiations of used STL types
+namespace {
+
+    stl_like_class<int> stlc_1;
+
+} // unnamed namespace
+
+
+// comps for int only to allow testing: normal use of vector is looping over a
+// range-checked version of __getitem__
+#if defined(__clang__) && defined(__APPLE__)
+namespace std {
+#define ns_prefix std::
+#elif defined(__GNUC__) || defined(__GNUG__)
+namespace __gnu_cxx {
+#define ns_prefix
+#endif
+extern template bool ns_prefix operator==(const std::vector<int>::iterator&,
+                         const std::vector<int>::iterator&);
+extern template bool ns_prefix operator!=(const std::vector<int>::iterator&,
+                         const std::vector<int>::iterator&);
+}
+
+
+//- helpers for testing array
+namespace ArrayTest {
+
+struct Point {
+    int px, py;
+};
+
+int get_pp_px(Point** p, int idx);
+int get_pp_py(Point** p, int idx);
+int get_pa_px(Point* p[], int idx);
+int get_pa_py(Point* p[], int idx);
+
+} // namespace ArrayTest
+
+
+// helpers for string testing
+extern std::string str_array_1[3];
+extern std::string str_array_2[];
+extern std::string str_array_3[3][2];
+extern std::string str_array_4[4][2][2];
diff --git a/pypy/module/_cppyy/test/stltypes.xml b/pypy/module/_cppyy/test/stltypes.xml
--- a/pypy/module/_cppyy/test/stltypes.xml
+++ b/pypy/module/_cppyy/test/stltypes.xml
@@ -1,10 +1,13 @@
 <lcgdict>
 
-  <namespace name="std" />
+  <class name="just_a_class" />
+  <class name="stringy_class" />
+  <class pattern="stl_like_class<*>" />
 
-  <class name="just_a_class" />
+  <namespace name="ArrayTest" />
+  <class pattern="ArrayTest::*" />
+  <function pattern="ArrayTest::*" />
 
-  <class name="std::string" />
-  <class name="stringy_class" />
+  <variable pattern="str_array_*" />
 
 </lcgdict>
diff --git a/pypy/module/_cppyy/test/test_advancedcpp.py b/pypy/module/_cppyy/test/test_advancedcpp.py
--- a/pypy/module/_cppyy/test/test_advancedcpp.py
+++ b/pypy/module/_cppyy/test/test_advancedcpp.py
@@ -56,6 +56,12 @@
             assert d.m_b ==  t(4)
             assert d.m_c ==  t(5)
             d.__destruct__()
+
+            defaulter_func = getattr(cppyy.gbl, '%s_defaulter_func' %n)
+            answers = [11, 22, 33, 3]
+            for idx in range(4):
+                assert defaulter_func(idx) == answers[idx]
+
         test_defaulter('short',  int)
         test_defaulter('ushort', int)
         test_defaulter('int',    int)
@@ -67,6 +73,13 @@
         test_defaulter('float',  float)
         test_defaulter('double', float)
 
+        assert cppyy.gbl.string_defaulter_func(0)               == "aap"
+        assert cppyy.gbl.string_defaulter_func(0, "zus")        == "zus"
+        assert cppyy.gbl.string_defaulter_func(1)               == "noot"
+        assert cppyy.gbl.string_defaulter_func(1, "zus")        == "noot"
+        assert cppyy.gbl.string_defaulter_func(1, "zus", "jet") == "jet"
+        assert cppyy.gbl.string_defaulter_func(2)               == "mies"
+
     def test02_simple_inheritance(self):
         """Test binding of a basic inheritance structure"""
 
@@ -148,6 +161,8 @@
         assert gbl.a_ns.d_ns.e_class.f_class.s_f      == 66
         assert gbl.a_ns.d_ns.e_class.f_class().m_f    == -6
 
+        raises(TypeError, gbl.a_ns)
+
     def test03a_namespace_lookup_on_update(self):
         """Test whether namespaces can be shared across dictionaries."""
 
@@ -474,6 +489,23 @@
         d2.__destruct__()
         d1.__destruct__()
 
+        RTS = cppyy.gbl.refers_to_self
+
+        r1 = RTS()
+        r2 = RTS()
+        r1.m_other = r2
+
+        r3 = r1.m_other
+        r4 = r1.m_other
+        assert r3 is r4
+
+        assert r3 == r2
+        assert r3 is r2
+
+        r3.extra = 42
+        assert r2.extra == 42
+        assert r4.extra == 42
+
     def test11_multi_methods(self):
         """Test calling of methods from multiple inheritance"""
 
@@ -653,10 +685,18 @@
 
         assert cppyy.gbl.my_global_double == 12.
         assert len(cppyy.gbl.my_global_array) == 500
-        assert cppyy.gbl.my_global_string == "aap  noot  mies"
+        assert cppyy.gbl.my_global_string1 == "aap  noot  mies"
+        return     # next line currently crashes
+        assert cppyy.gbl.my_global_string2 == "zus jet teun"
         # TODO: currently fails b/c double** not understood as &double*
         #assert cppyy.gbl.my_global_ptr[0] == 1234.
 
+        v = cppyy.gbl.my_global_int_holders
+        assert len(v) == 5
+        expected_vals = [13, 42, 88, -1, 17]
+        for i in range(len(v)):
+            assert v[i].m_val == expected_vals[i]
+
     def test22_exceptions(self):
         """Catching of C++ exceptions"""
 
@@ -675,3 +715,35 @@
             t.throw_exception()
         except Exception as e:
             "C++ function failed" in str(e)
+
+    def test23_using(self):
+        """Accessibility of using declarations"""
+
+        import _cppyy as cppyy
+
+        assert cppyy.gbl.UsingBase().vcheck() == 'A'
+
+        B = cppyy.gbl.UsingDerived
+        assert not 'UsingBase' in B.__init__.__doc__
+
+        b1 = B()
+        assert b1.m_int    == 13
+        assert b1.m_int2   == 42
+        assert b1.vcheck() == 'B'
+ 
+        b2 = B(10)
+        assert b2.m_int    == 10
+        assert b2.m_int2   == 42
+        assert b2.vcheck() == 'B'
+
+        b3 = B(b2)
+        assert b3.m_int    == 10
+        assert b3.m_int2   == 42
+        assert b3.vcheck() == 'B'
+
+    def test24_typedef_to_private_class(self):
+        """Typedefs to private classes should not resolve"""
+
+        import _cppyy as cppyy
+
+        assert cppyy.gbl.TypedefToPrivateClass().f().m_val == 42
diff --git a/pypy/module/_cppyy/test/test_datatypes.py b/pypy/module/_cppyy/test/test_datatypes.py
--- a/pypy/module/_cppyy/test/test_datatypes.py
+++ b/pypy/module/_cppyy/test/test_datatypes.py
@@ -181,7 +181,7 @@
         names = ['uchar', 'short', 'ushort', 'int', 'uint', 'long', 'ulong']
         import array
         a = range(self.N)
-        atypes = ['B', 'h', 'H', 'i', 'I', 'l', 'L' ]
+        atypes = ['B', 'h', 'H', 'i', 'I', 'l', 'L']
         for j in range(len(names)):
             b = array.array(atypes[j], a)
             setattr(c, 'm_'+names[j]+'_array', b)     # buffer copies
@@ -189,6 +189,7 @@
                 assert eval('c.m_%s_array[i]' % names[j]) == b[i]
 
             setattr(c, 'm_'+names[j]+'_array2', b)    # pointer copies
+            assert 3 < self.N
             b[3] = 28
             for i in range(self.N):
                 assert eval('c.m_%s_array2[i]' % names[j]) == b[i]
@@ -210,7 +211,7 @@
 
         a = range(self.N)
         # test arrays in mixed order, to give overload resolution a workout
-        for t in ['d', 'i', 'f', 'H', 'I', 'h', 'L', 'l' ]:
+        for t in ['d', 'i', 'f', 'H', 'I', 'h', 'L', 'l']:
             b = array.array(t, a)
 
             # typed passing
@@ -689,9 +690,6 @@
                      'get_long_array',   'get_long_array2',
                      'get_ulong_array',  'get_ulong_array2']:
             arr = getattr(c, func)()
-            arr = arr.shape.fromaddress(arr.itemaddress(0), self.N)
-
-            """TODO: interface change ...
             arr.reshape((self.N,))
             assert len(arr) == self.N
 
@@ -703,7 +701,7 @@
 
             l = list(arr)
             for i in range(self.N):
-                assert arr[i] == l[i]"""
+                assert arr[i] == l[i]
 
     def test20_voidp(self):
         """Test usage of void* data"""
@@ -756,6 +754,15 @@
     def test21_function_pointers(self):
         """Function pointer passing"""
 
+        import os
+
+        # TODO: currently crashes if fast path disabled
+        try:
+            if os.environ['CPPYY_DISABLE_FASTPATH']:
+                return
+        except KeyError:
+            pass
+
         import _cppyy as cppyy
 
         f1 = cppyy.gbl.sum_of_int
diff --git a/pypy/module/_cppyy/test/test_overloads.py b/pypy/module/_cppyy/test/test_overloads.py
--- a/pypy/module/_cppyy/test/test_overloads.py
+++ b/pypy/module/_cppyy/test/test_overloads.py
@@ -58,15 +58,19 @@
         c = c_overload()
         raises(TypeError, c.__dispatch__, 'get_int', 12)
         raises(LookupError, c.__dispatch__, 'get_int', 'does_not_exist')
-        assert c.__dispatch__('get_int', 'a_overload*')(a_overload()) == 42
-        assert c.__dispatch__('get_int', 'b_overload*')(b_overload()) == 13
+        assert c.__dispatch__('get_int', 'a_overload*')(a_overload())             == 42
+        assert c_overload.get_int.__overload__('a_overload*')(c, a_overload())    == 42
+        assert c.__dispatch__('get_int', 'b_overload*')(b_overload())             == 13
+        assert c_overload.get_int.__overload__('b_overload*')(c, b_overload())    == 13
 
         assert c_overload().__dispatch__('get_int', 'a_overload*')(a_overload())  == 42
         # TODO: #assert c_overload.__dispatch__('get_int', 'b_overload*')(c, b_overload()) == 13
 
         d = d_overload()
-        assert d.__dispatch__('get_int', 'a_overload*')(a_overload()) == 42
-        assert d.__dispatch__('get_int', 'b_overload*')(b_overload()) == 13
+        assert d.__dispatch__('get_int', 'a_overload*')(a_overload())             == 42
+        assert d_overload.get_int.__overload__('a_overload*')(d, a_overload())    == 42
+        assert d.__dispatch__('get_int', 'b_overload*')(b_overload())             == 13
+        assert d_overload.get_int.__overload__('b_overload*')(d, b_overload())    == 13
 
         nb = ns_a_overload.b_overload()
         raises(TypeError, nb.f, c_overload())
diff --git a/pypy/module/_cppyy/test/test_stltypes.py b/pypy/module/_cppyy/test/test_stltypes.py
--- a/pypy/module/_cppyy/test/test_stltypes.py
+++ b/pypy/module/_cppyy/test/test_stltypes.py
@@ -22,12 +22,12 @@
     def test01_builtin_type_vector_types(self):
         """Test access to std::vector<int>/std::vector<double>"""
 
-        import _cppyy
+        import _cppyy as cppyy
 
-        assert _cppyy.gbl.std        is _cppyy.gbl.std
-        assert _cppyy.gbl.std.vector is _cppyy.gbl.std.vector
+        assert cppyy.gbl.std        is cppyy.gbl.std
+        assert cppyy.gbl.std.vector is cppyy.gbl.std.vector
 
-        assert callable(_cppyy.gbl.std.vector)
+        assert callable(cppyy.gbl.std.vector)
 
         type_info = (
             ("int",     int),
@@ -36,14 +36,14 @@
         )
 
         for c_type, p_type in type_info:
-            tv1 = getattr(_cppyy.gbl.std, 'vector<%s>' % c_type)
-            tv2 = _cppyy.gbl.std.vector(p_type)
+            tv1 = getattr(cppyy.gbl.std, 'vector<%s>' % c_type)
+            tv2 = cppyy.gbl.std.vector(p_type)
             assert tv1 is tv2
-            assert tv1.iterator is _cppyy.gbl.std.vector(p_type).iterator
+            assert tv1.iterator is cppyy.gbl.std.vector(p_type).iterator
 
             #----- 
-            v = tv1(); v += range(self.N)    # default args from Reflex are useless :/
-            if p_type == int:                # only type with == and != reflected in .xml
+            v = tv1(); v += range(self.N)
+            if p_type == int:
                 assert v.begin().__eq__(v.begin())
                 assert v.begin() == v.begin()
                 assert v.end() == v.end()
@@ -58,6 +58,7 @@
 
             assert v.size() == self.N
             assert len(v) == self.N
+            assert len(v.data()) == self.N
 
             #-----
             v = tv1()
@@ -73,16 +74,16 @@
     def test02_user_type_vector_type(self):
         """Test access to an std::vector<just_a_class>"""
 
-        import _cppyy
+        import _cppyy as cppyy
 
-        assert _cppyy.gbl.std        is _cppyy.gbl.std
-        assert _cppyy.gbl.std.vector is _cppyy.gbl.std.vector
+        assert cppyy.gbl.std        is cppyy.gbl.std
+        assert cppyy.gbl.std.vector is cppyy.gbl.std.vector
 
-        assert callable(_cppyy.gbl.std.vector)
+        assert callable(cppyy.gbl.std.vector)
 
-        tv1 = getattr(_cppyy.gbl.std, 'vector<just_a_class>')
-        tv2 = _cppyy.gbl.std.vector('just_a_class')
-        tv3 = _cppyy.gbl.std.vector(_cppyy.gbl.just_a_class)
+        tv1 = getattr(cppyy.gbl.std, 'vector<just_a_class>')
+        tv2 = cppyy.gbl.std.vector('just_a_class')
+        tv3 = cppyy.gbl.std.vector(cppyy.gbl.just_a_class)
 
         assert tv1 is tv2
         assert tv2 is tv3
@@ -95,7 +96,7 @@
         assert hasattr(v, 'end' )
 
         for i in range(self.N):
-            v.push_back(_cppyy.gbl.just_a_class())
+            v.push_back(cppyy.gbl.just_a_class())
             v[i].m_i = i
             assert v[i].m_i == i
 
@@ -105,9 +106,9 @@
     def test03_empty_vector_type(self):
         """Test behavior of empty std::vector<int>"""
 
-        import _cppyy
+        import _cppyy as cppyy
 
-        v = _cppyy.gbl.std.vector(int)()
+        v = cppyy.gbl.std.vector(int)()
         for arg in v:
             pass
         v.__destruct__()
@@ -115,9 +116,9 @@
     def test04_vector_iteration(self):
         """Test iteration over an std::vector<int>"""
 
-        import _cppyy
+        import _cppyy as cppyy
 
-        v = _cppyy.gbl.std.vector(int)()
+        v = cppyy.gbl.std.vector(int)()
 
         for i in range(self.N):
             v.push_back(i)
@@ -140,9 +141,9 @@
     def test05_push_back_iterables_with_iadd(self):
         """Test usage of += of iterable on push_back-able container"""
 
-        import _cppyy
+        import _cppyy as cppyy
 
-        v = _cppyy.gbl.std.vector(int)()
+        v = cppyy.gbl.std.vector(int)()
 
         v += [1, 2, 3]
         assert len(v) == 3
@@ -159,7 +160,7 @@
         raises(TypeError, v.__iadd__, (7, '8'))  # string shouldn't pass
         assert len(v) == 7   # TODO: decide whether this should roll-back
 
-        v2 = _cppyy.gbl.std.vector(int)()
+        v2 = cppyy.gbl.std.vector(int)()
         v2 += [8, 9]
         assert len(v2) == 2
         assert v2[0] == 8
@@ -174,9 +175,9 @@
     def test06_vector_indexing(self):
         """Test python-style indexing to an std::vector<int>"""
 
-        import _cppyy
+        import _cppyy as cppyy
 
-        v = _cppyy.gbl.std.vector(int)()
+        v = cppyy.gbl.std.vector(int)()
 
         for i in range(self.N):
             v.push_back(i)
@@ -209,9 +210,9 @@
     def test01_string_argument_passing(self):
         """Test mapping of python strings and std::string"""
 
-        import _cppyy
-        std = _cppyy.gbl.std
-        stringy_class = _cppyy.gbl.stringy_class
+        import _cppyy as cppyy
+        std = cppyy.gbl.std
+        stringy_class = cppyy.gbl.stringy_class
 
         c, s = stringy_class(""), std.string("test1")
 
@@ -240,9 +241,9 @@
     def test02_string_data_access(self):
         """Test access to std::string object data members"""
 
-        import _cppyy
-        std = _cppyy.gbl.std
-        stringy_class = _cppyy.gbl.stringy_class
+        import _cppyy as cppyy
+        std = cppyy.gbl.std
+        stringy_class = cppyy.gbl.stringy_class
 
         c, s = stringy_class("dummy"), std.string("test string")
 
@@ -261,9 +262,9 @@
 
         return # don't bother; is fixed in cling-support
 
-        import _cppyy
-        std = _cppyy.gbl.std
-        stringy_class = _cppyy.gbl.stringy_class
+        import _cppyy as cppyy
+        std = cppyy.gbl.std
+        stringy_class = cppyy.gbl.stringy_class
 
         t0 = "aap\0noot"
         assert t0 == "aap\0noot"
@@ -288,8 +289,8 @@
     def test01_builtin_list_type(self):
         """Test access to a list<int>"""
 
-        import _cppyy
-        std = _cppyy.gbl.std
+        import _cppyy as cppyy
+        std = cppyy.gbl.std
 
         type_info = (
             ("int",     int),
@@ -299,9 +300,9 @@
 
         for c_type, p_type in type_info:
             tl1 = getattr(std, 'list<%s>' % c_type)
-            tl2 = _cppyy.gbl.std.list(p_type)
+            tl2 = cppyy.gbl.std.list(p_type)
             assert tl1 is tl2
-            assert tl1.iterator is _cppyy.gbl.std.list(p_type).iterator
+            assert tl1.iterator is cppyy.gbl.std.list(p_type).iterator
 
             #-----
             a = tl1()
@@ -323,8 +324,8 @@
     def test02_empty_list_type(self):
         """Test behavior of empty list<int>"""
 
-        import _cppyy
-        std = _cppyy.gbl.std
+        import _cppyy as cppyy
+        std = cppyy.gbl.std
 
         a = std.list(int)()
         for arg in a:
@@ -344,8 +345,8 @@
     def test01_builtin_map_type(self):
         """Test access to a map<int,int>"""
 
-        import _cppyy
-        std = _cppyy.gbl.std
+        import _cppyy as cppyy
+        std = cppyy.gbl.std
 
         a = std.map(int, int)()
         for i in range(self.N):
@@ -373,8 +374,8 @@
     def test02_keyed_maptype(self):
         """Test access to a map<std::string,int>"""
 
-        import _cppyy
-        std = _cppyy.gbl.std
+        import _cppyy as cppyy
+        std = cppyy.gbl.std
 
         a = std.map(std.string, int)()
         for i in range(self.N):
@@ -386,8 +387,8 @@
     def test03_empty_maptype(self):
         """Test behavior of empty map<int,int>"""
 
-        import _cppyy
-        std = _cppyy.gbl.std
+        import _cppyy as cppyy
+        std = cppyy.gbl.std
 
         m = std.map(int, int)()
         for key, value in m:
@@ -396,8 +397,9 @@
     def test04_unsignedvalue_typemap_types(self):
         """Test assignability of maps with unsigned value types"""
 
-        import _cppyy, math, sys
-        std = _cppyy.gbl.std
+        import _cppyy as cppyy
+        import math, sys
+        std = cppyy.gbl.std
 
         mui = std.map(str, 'unsigned int')()
         mui['one'] = 1
@@ -420,8 +422,8 @@
     def test05_STL_like_class_indexing_overloads(self):
         """Test overloading of operator[] in STL like class"""
 
-        import _cppyy
-        stl_like_class = _cppyy.gbl.stl_like_class
+        import _cppyy as cppyy
+        stl_like_class = cppyy.gbl.stl_like_class
 
         a = stl_like_class(int)()
         assert a["some string" ] == 'string'
@@ -430,8 +432,8 @@
     def test06_STL_like_class_iterators(self):
         """Test the iterator protocol mapping for an STL like class"""
 
-        import _cppyy
-        stl_like_class = _cppyy.gbl.stl_like_class
+        import _cppyy as cppyy
+        stl_like_class = cppyy.gbl.stl_like_class
 
         a = stl_like_class(int)()
         for i in a:
@@ -452,8 +454,8 @@
     def test01_builtin_vector_iterators(self):
         """Test iterator comparison with operator== reflected"""
 
-        import _cppyy
-        std = _cppyy.gbl.std
+        import _cppyy as cppyy
+        std = cppyy.gbl.std
 
         v = std.vector(int)()
         v.resize(1)
@@ -489,9 +491,9 @@
     def test01_explicit_templates(self):
         """Explicit use of Template class"""
 
-        import _cppyy
+        import _cppyy as cppyy
 
-        vector = _cppyy.Template('vector', _cppyy.gbl.std)
+        vector = cppyy.Template('vector', cppyy.gbl.std)
         assert vector[int] == vector(int)
 
         v = vector[int]()
@@ -501,3 +503,54 @@
         assert len(v) == N
         for i in range(N):
             assert v[i] == i
+
+
+class AppTestSTLARRAY:
+    spaceconfig = dict(usemodules=['_cppyy', '_rawffi', 'itertools'])
+
+    def setup_class(cls):
+        cls.w_test_dct  = cls.space.newtext(test_dct)
+        cls.w_stlarray = cls.space.appexec([], """():
+            import ctypes, _cppyy
+            _cppyy._post_import_startup()
+            return ctypes.CDLL(%r, ctypes.RTLD_GLOBAL)""" % (test_dct, ))
+
+    def test01_array_of_basic_types(self):
+        """Usage of std::array of basic types"""
+
+        import _cppyy as cppyy
+        std = cppyy.gbl.std
+
+        a = std.array[int, 4]()
+        assert len(a) == 4
+        for i in range(len(a)):
+            a[i] = i
+            assert a[i] == i
+
+    def test02_array_of_pods(self):
+        """Usage of std::array of PODs"""
+
+        import _cppyy as cppyy
+        gbl, std = cppyy.gbl, cppyy.gbl.std
+
+        a = std.array[gbl.ArrayTest.Point, 4]()
+        assert len(a) == 4
+        for i in range(len(a)):
+            a[i].px = i
+            assert a[i].px == i
+            a[i].py = i**2
+            assert a[i].py == i**2
+
+    def test03_array_of_pointer_to_pods(self):
+        """Usage of std::array of pointer to PODs"""
+
+        import cppyy
+        from cppyy import gbl
+        from cppyy.gbl import std
+
+        ll = [gbl.ArrayTest.Point() for i in range(4)]
+        for i in range(len(ll)):
+            ll[i].px = 13*i
+            ll[i].py = 42*i
+
+        # more tests in cppyy/test/test_stltypes.py, but currently not supported
diff --git a/pypy/module/cpyext/cdatetime.py b/pypy/module/cpyext/cdatetime.py
--- a/pypy/module/cpyext/cdatetime.py
+++ b/pypy/module/cpyext/cdatetime.py
@@ -12,18 +12,19 @@
 from pypy.module.__pypy__.interp_pypydatetime import (W_DateTime_Date,
     W_DateTime_Time, W_DateTime_Delta)
 from rpython.tool.sourcetools import func_renamer
+from pypy.module.cpyext.state import State
 
 cts.parse_header(parse_dir / 'cpyext_datetime.h')
 
 
 PyDateTime_CAPI = cts.gettype('PyDateTime_CAPI')
 
-datetimeAPI_global = []
 
 @cpython_api([], lltype.Ptr(PyDateTime_CAPI))
 def _PyDateTime_Import(space):
-    if len(datetimeAPI_global) >0:
-        return datetimeAPI_global[0]
+    state = space.fromcache(State)
+    if len(state.datetimeAPI) > 0:
+        return state.datetimeAPI[0]
     datetimeAPI = lltype.malloc(PyDateTime_CAPI, flavor='raw',
                                 track_allocation=False)
 
@@ -66,8 +67,8 @@
         _PyDelta_FromDelta.api_func.functype,
         _PyDelta_FromDelta.api_func.get_wrapper(space))
 
-    datetimeAPI_global.append(datetimeAPI)
-    return datetimeAPI
+    state.datetimeAPI.append(datetimeAPI)
+    return state.datetimeAPI[0]
 
 PyDateTime_Time = cts.gettype('PyDateTime_Time*')
 PyDateTime_DateTime = cts.gettype('PyDateTime_DateTime*')
@@ -120,21 +121,10 @@
                    dealloc=type_dealloc,
                   )
 
-    # why do we need date_dealloc? Since W_DateTime_Date is the base class for
-    # app level datetime.date. If a c-extension class uses datetime.date for its
-    # base class and defines a tp_dealloc, we will get this:
-    # c_class->tp_dealloc == tp_dealloc_func
-    # c_class->tp_base == datetime.date, 
-    #                     datetime.date->tp_dealloc = _PyPy_subtype_dealloc
-    # datetime.date->tp_base = W_DateTime_Date
-    #                    W_DateTime_Date->tp_dealloc = _PyPy_subtype_dealloc
-    # but _PyPy_subtype_dealloc will call tp_dealloc_func, which can call its
-    # base's tp_dealloc and we get recursion. So break the recursion by setting
-    # W_DateTime_Date->tp_dealloc
     make_typedescr(W_DateTime_Date.typedef,
                    basestruct=PyDateTime_DateTime.TO,
                    attach=type_attach,
-                   dealloc=date_dealloc,
+                   dealloc=type_dealloc,
                   )
 
     make_typedescr(W_DateTime_Delta.typedef,
@@ -144,30 +134,45 @@
 
 def type_attach(space, py_obj, w_obj, w_userdata=None):
     '''Fills a newly allocated py_obj from the w_obj
+    If it is a datetime.time or datetime.datetime, it may have tzinfo
     '''
-    if space.type(w_obj).name == 'date':
-        # No tzinfo
-        return
-    py_datetime = rffi.cast(PyDateTime_Time, py_obj)
-    w_tzinfo = space.getattr(w_obj, space.newtext('tzinfo'))
-    if space.is_none(w_tzinfo):
-        py_datetime.c_hastzinfo = cts.cast('unsigned char', 0)
-        py_datetime.c_tzinfo = lltype.nullptr(PyObject.TO)
-    else:
-        py_datetime.c_hastzinfo = cts.cast('unsigned char', 1)
-        py_datetime.c_tzinfo = make_ref(space, w_tzinfo)
+    state = space.fromcache(State)
+    # cannot raise here, so just crash
+    assert len(state.datetimeAPI) > 0
+    if state.datetimeAPI[0].c_TimeType == py_obj.c_ob_type:
+        py_datetime = rffi.cast(PyDateTime_Time, py_obj)
+        w_tzinfo = space.getattr(w_obj, space.newtext('tzinfo'))
+        if space.is_none(w_tzinfo):
+            py_datetime.c_hastzinfo = cts.cast('unsigned char', 0)
+            py_datetime.c_tzinfo = lltype.nullptr(PyObject.TO)
+        else:
+            py_datetime.c_hastzinfo = cts.cast('unsigned char', 1)
+            py_datetime.c_tzinfo = make_ref(space, w_tzinfo)
+    elif state.datetimeAPI[0].c_DateTimeType == py_obj.c_ob_type:
+        # For now this is exactly the same structure as PyDateTime_Time
+        py_datetime = rffi.cast(PyDateTime_DateTime, py_obj)
+        w_tzinfo = space.getattr(w_obj, space.newtext('tzinfo'))
+        if space.is_none(w_tzinfo):
+            py_datetime.c_hastzinfo = cts.cast('unsigned char', 0)
+            py_datetime.c_tzinfo = lltype.nullptr(PyObject.TO)
+        else:
+            py_datetime.c_hastzinfo = cts.cast('unsigned char', 1)
+            py_datetime.c_tzinfo = make_ref(space, w_tzinfo)
 
 @slot_function([PyObject], lltype.Void)
 def type_dealloc(space, py_obj):
-    py_datetime = rffi.cast(PyDateTime_Time, py_obj)
-    if (widen(py_datetime.c_hastzinfo) != 0):
-        decref(space, py_datetime.c_tzinfo)
     from pypy.module.cpyext.object import _dealloc
-    _dealloc(space, py_obj)
-
- at slot_function([PyObject], lltype.Void)
-def date_dealloc(space, py_obj):
-    from pypy.module.cpyext.object import _dealloc
+    state = space.fromcache(State)
+    # cannot raise here, so just crash
+    assert len(state.datetimeAPI) > 0
+    if state.datetimeAPI[0].c_TimeType == py_obj.c_ob_type:
+        py_datetime = rffi.cast(PyDateTime_Time, py_obj)
+        if (widen(py_datetime.c_hastzinfo) != 0):
+            decref(space, py_datetime.c_tzinfo)
+    elif state.datetimeAPI[0].c_DateTimeType == py_obj.c_ob_type:
+        py_datetime = rffi.cast(PyDateTime_DateTime, py_obj)
+        if (widen(py_datetime.c_hastzinfo) != 0):
+            decref(space, py_datetime.c_tzinfo)
     _dealloc(space, py_obj)
 
 def timedeltatype_attach(space, py_obj, w_obj, w_userdata=None):
diff --git a/pypy/module/cpyext/state.py b/pypy/module/cpyext/state.py
--- a/pypy/module/cpyext/state.py
+++ b/pypy/module/cpyext/state.py
@@ -42,6 +42,8 @@
         # A mapping {filename: copy-of-the-w_dict}, similar to CPython's
         # variable 'extensions' in Python/import.c.
         self.extensions = {}
+        # XXX will leak if _PyDateTime_Import already called
+        self.datetimeAPI = []
 
     def set_exception(self, operror):
         self.clear_exception()
diff --git a/pypy/module/cpyext/test/test_datetime.py b/pypy/module/cpyext/test/test_datetime.py
--- a/pypy/module/cpyext/test/test_datetime.py
+++ b/pypy/module/cpyext/test/test_datetime.py
@@ -307,18 +307,20 @@
         from datetime import tzinfo, datetime, timedelta, time
         # copied from datetime documentation
         class GMT1(tzinfo):
-           def utcoffset(self, dt):
-               return timedelta(hours=1) + self.dst(dt)
-           def dst(self, dt):
-               return timedelta(0)
-           def tzname(self,dt):
+            def __del__(self):
+                print 'deleting GMT1'
+            def utcoffset(self, dt):
+                return timedelta(hours=1) + self.dst(dt)
+            def dst(self, dt):
+                return timedelta(0)
+            def tzname(self,dt):
                 return "GMT +1"
         gmt1 = GMT1()
         dt1 = module.time_with_tzinfo(gmt1)
         assert dt1 == time(6, 6, 6, 6, gmt1)
         assert '+01' in str(dt1)
-        assert module.datetime_with_tzinfo(gmt1) == datetime(
-            2000, 6, 6, 6, 6, 6, 6, gmt1)
+        dt_tz = module.datetime_with_tzinfo(gmt1)
+        assert dt_tz == datetime(2000, 6, 6, 6, 6, 6, 6, gmt1)
 
     def test_checks(self):
         module = self.import_extension('foo', [

From pypy.commits at gmail.com  Tue Jul 17 02:01:36 2018
From: pypy.commits at gmail.com (wlav)
Date: Mon, 16 Jul 2018 23:01:36 -0700 (PDT)
Subject: [pypy-commit] pypy cppyy-packaging: more settable overload
 properties
Message-ID: <5b4d8640.1c69fb81.9fb68.1207@mx.google.com>

Author: Wim Lavrijsen <WLavrijsen at lbl.gov>
Branch: cppyy-packaging
Changeset: r94868:c67865f23398
Date: 2018-07-15 20:17 -0700
http://bitbucket.org/pypy/pypy/changeset/c67865f23398/

Log:	more settable overload properties

diff --git a/pypy/module/_cppyy/interp_cppyy.py b/pypy/module/_cppyy/interp_cppyy.py
--- a/pypy/module/_cppyy/interp_cppyy.py
+++ b/pypy/module/_cppyy/interp_cppyy.py
@@ -24,6 +24,7 @@
 INSTANCE_FLAGS_IS_RVALUE   = 0x0004
 
 OVERLOAD_FLAGS_USE_FFI     = 0x0001
+OVERLOAD_FLAGS_CREATES     = 0x0002
 
 FUNCTION_IS_GLOBAL         = 0x0001
 FUNCTION_IS_STATIC         = 0x0001
@@ -458,6 +459,36 @@
 # need forwarding, which the normal instancemethod does not provide, hence this
 # derived class.
 class MethodWithProps(Method):
+    # set life management of result from the call
+    def fget_creates(self, space):
+        f = space.interp_w(W_CPPOverload, self.w_function)
+        return f.fget_creates(space)
+
+    @unwrap_spec(value=bool)
+    def fset_creates(self, space, value):
+        f = space.interp_w(W_CPPOverload, self.w_function)
+        f.fset_creates(space, value)
+
+    # set ownership policy of arguments (not yet implemented)
+    def fget_mempolicy(self, space):
+        f = space.interp_w(W_CPPOverload, self.w_function)
+        return f.fget_mempolicy(space)
+
+    @unwrap_spec(value=int)
+    def fset_mempolicy(self, space, value):
+        f = space.interp_w(W_CPPOverload, self.w_function)
+        f.fset_mempolicy(space, value)
+
+    # set to release the gil during call (not yet implemented)
+    def fget_release_gil(self, space):
+        f = space.interp_w(W_CPPOverload, self.w_function)
+        return f.fget_release_gil(space)
+
+    @unwrap_spec(value=bool)
+    def fset_release_gil(self, space, value):
+        f = space.interp_w(W_CPPOverload, self.w_function)
+        f.fset_release_gil(space, value)
+
     # allow user to determine ffi use rules per overload
     def fget_useffi(self, space):
         f = space.interp_w(W_CPPOverload, self.w_function)
@@ -473,22 +504,25 @@
     __doc__ = """cpp_instancemethod(function, instance, class)
 
 Create an instance method object.""",
-    __new__ = interp2app(MethodWithProps.descr_method__new__.im_func),
-    __call__ = interp2app(MethodWithProps.descr_method_call),
-    __get__ = interp2app(MethodWithProps.descr_method_get),
-    im_func = interp_attrproperty_w('w_function', cls=MethodWithProps),
-    __func__ = interp_attrproperty_w('w_function', cls=MethodWithProps),
-    im_self = interp_attrproperty_w('w_instance', cls=MethodWithProps),
-    __self__ = interp_attrproperty_w('w_instance', cls=MethodWithProps),
-    im_class = interp_attrproperty_w('w_class', cls=MethodWithProps),
+    __new__          = interp2app(MethodWithProps.descr_method__new__.im_func),
+    __call__         = interp2app(MethodWithProps.descr_method_call),
+    __get__          = interp2app(MethodWithProps.descr_method_get),
+    im_func          = interp_attrproperty_w('w_function', cls=MethodWithProps),
+    __func__         = interp_attrproperty_w('w_function', cls=MethodWithProps),
+    im_self          = interp_attrproperty_w('w_instance', cls=MethodWithProps),
+    __self__         = interp_attrproperty_w('w_instance', cls=MethodWithProps),
+    im_class         = interp_attrproperty_w('w_class', cls=MethodWithProps),
     __getattribute__ = interp2app(MethodWithProps.descr_method_getattribute),
-    __eq__ = interp2app(MethodWithProps.descr_method_eq),
-    __ne__ = descr_generic_ne,
-    __hash__ = interp2app(MethodWithProps.descr_method_hash),
-    __repr__ = interp2app(MethodWithProps.descr_method_repr),
-    __reduce__ = interp2app(MethodWithProps.descr_method__reduce__),
-    __weakref__ = make_weakref_descr(MethodWithProps),
-    __useffi__ = GetSetProperty(MethodWithProps.fget_useffi, MethodWithProps.fset_useffi),
+    __eq__           = interp2app(MethodWithProps.descr_method_eq),
+    __ne__           = descr_generic_ne,
+    __hash__         = interp2app(MethodWithProps.descr_method_hash),
+    __repr__         = interp2app(MethodWithProps.descr_method_repr),
+    __reduce__       = interp2app(MethodWithProps.descr_method__reduce__),
+    __weakref__      = make_weakref_descr(MethodWithProps),
+    __creates__      = GetSetProperty(MethodWithProps.fget_creates,     MethodWithProps.fset_creates),
+    __mempolicy__    = GetSetProperty(MethodWithProps.fget_mempolicy,   MethodWithProps.fset_mempolicy),
+    __release_gil__  = GetSetProperty(MethodWithProps.fget_release_gil, MethodWithProps.fset_release_gil),
+    __useffi__       = GetSetProperty(MethodWithProps.fget_useffi,      MethodWithProps.fset_useffi),
     )
 MethodWithProps.typedef.acceptable_as_base_class = False
 
@@ -551,7 +585,12 @@
         for i in range(len(self.functions)):
             cppyyfunc = self.functions[i]
             try:
-                return cppyyfunc.call(cppthis, args_w, self.flags & OVERLOAD_FLAGS_USE_FFI)
+                w_result = cppyyfunc.call(cppthis, args_w, self.flags & OVERLOAD_FLAGS_USE_FFI)
+                if self.flags & OVERLOAD_FLAGS_CREATES:
+                    if isinstance(w_result, W_CPPInstance):
+                        cppinstance = self.space.interp_w(W_CPPInstance, w_result)
+                        cppinstance.fset_python_owns(self.space, self.space.w_True)
+                return w_result
             except Exception:
                 pass
 
@@ -564,6 +603,7 @@
         for i in range(len(self.functions)):
             cppyyfunc = self.functions[i]
             try:
+                # no need to set ownership on the return value, as none of the methods execute
                 return cppyyfunc.call(cppthis, args_w, self.flags & OVERLOAD_FLAGS_USE_FFI)
             except OperationError as e:
                 # special case if there's just one function, to prevent clogging the error message
@@ -602,6 +642,33 @@
                 return W_CPPOverload(self.space, self.scope, [f])
         raise oefmt(self.space.w_LookupError, "signature '%s' not found", signature)
 
+    # set life management of result from the call
+    def fget_creates(self, space):
+        return space.newbool(bool(self.flags & OVERLOAD_FLAGS_CREATES))
+
+    @unwrap_spec(value=bool)
+    def fset_creates(self, space, value):
+        if space.is_true(value):
+            self.flags |= OVERLOAD_FLAGS_CREATES
+        else:
+            self.flags &= ~OVERLOAD_FLAGS_CREATES
+
+    # set ownership policy of arguments (not yet implemented)
+    def fget_mempolicy(self, space):
+        return space.newint(0)
+
+    @unwrap_spec(value=int)
+    def fset_mempolicy(self, space, value):
+        pass
+
+    # set to release the gil during call (not yet implemented)
+    def fget_release_gil(self, space):
+        return space.newbool(True)
+
+    @unwrap_spec(value=bool)
+    def fset_release_gil(self, space, value):
+        pass
+
     # allow user to determine ffi use rules per overload
     def fget_useffi(self, space):
         return space.newbool(bool(self.flags & OVERLOAD_FLAGS_USE_FFI))
@@ -625,11 +692,14 @@
 
 W_CPPOverload.typedef = TypeDef(
     'CPPOverload',
-    __get__      = interp2app(W_CPPOverload.descr_get),
-    __call__     = interp2app(W_CPPOverload.call_args),
-    __useffi__   = GetSetProperty(W_CPPOverload.fget_useffi, W_CPPOverload.fset_useffi),
-    __overload__ = interp2app(W_CPPOverload.mp_overload),
-    __doc__      = GetSetProperty(W_CPPOverload.fget_doc)
+    __get__         = interp2app(W_CPPOverload.descr_get),
+    __call__        = interp2app(W_CPPOverload.call_args),
+    __creates__     = GetSetProperty(W_CPPOverload.fget_creates,     W_CPPOverload.fset_creates),
+    __mempolicy__   = GetSetProperty(W_CPPOverload.fget_mempolicy,   W_CPPOverload.fset_mempolicy),
+    __release_gil__ = GetSetProperty(W_CPPOverload.fget_release_gil, W_CPPOverload.fset_release_gil),
+    __useffi__      = GetSetProperty(W_CPPOverload.fget_useffi,      W_CPPOverload.fset_useffi),
+    __overload__    = interp2app(W_CPPOverload.mp_overload),
+    __doc__         = GetSetProperty(W_CPPOverload.fget_doc)
 )
 
 
@@ -658,11 +728,14 @@
 
 W_CPPStaticOverload.typedef = TypeDef(
     'CPPStaticOverload',
-    __get__      = interp2app(W_CPPStaticOverload.descr_get),
-    __call__     = interp2app(W_CPPStaticOverload.call_args),
-    __useffi__   = GetSetProperty(W_CPPStaticOverload.fget_useffi, W_CPPStaticOverload.fset_useffi),
-    __overload__ = interp2app(W_CPPStaticOverload.mp_overload),
-    __doc__      = GetSetProperty(W_CPPStaticOverload.fget_doc)
+    __get__         = interp2app(W_CPPStaticOverload.descr_get),
+    __call__        = interp2app(W_CPPStaticOverload.call_args),
+    __creates__     = GetSetProperty(W_CPPStaticOverload.fget_creates,     W_CPPStaticOverload.fset_creates),
+    __mempolicy__   = GetSetProperty(W_CPPStaticOverload.fget_mempolicy,   W_CPPStaticOverload.fset_mempolicy),
+    __release_gil__ = GetSetProperty(W_CPPStaticOverload.fget_release_gil, W_CPPStaticOverload.fset_release_gil),
+    __useffi__      = GetSetProperty(W_CPPStaticOverload.fget_useffi,      W_CPPStaticOverload.fset_useffi),
+    __overload__    = interp2app(W_CPPStaticOverload.mp_overload),
+    __doc__         = GetSetProperty(W_CPPStaticOverload.fget_doc)
 )
 
 
@@ -871,11 +944,14 @@
 
 W_CPPTemplateOverload.typedef = TypeDef(
     'CPPTemplateOverload',
-    __get__      = interp2app(W_CPPTemplateOverload.descr_get),
-    __getitem__  = interp2app(W_CPPTemplateOverload.getitem),
-    __call__     = interp2app(W_CPPTemplateOverload.call_args),
-    __useffi__   = GetSetProperty(W_CPPTemplateOverload.fget_useffi, W_CPPTemplateOverload.fset_useffi),
-    __doc__      = GetSetProperty(W_CPPTemplateOverload.fget_doc)
+    __get__         = interp2app(W_CPPTemplateOverload.descr_get),
+    __getitem__     = interp2app(W_CPPTemplateOverload.getitem),
+    __call__        = interp2app(W_CPPTemplateOverload.call_args),
+    __creates__     = GetSetProperty(W_CPPTemplateOverload.fget_creates,     W_CPPTemplateOverload.fset_creates),
+    __mempolicy__   = GetSetProperty(W_CPPTemplateOverload.fget_mempolicy,   W_CPPTemplateOverload.fset_mempolicy),
+    __release_gil__ = GetSetProperty(W_CPPTemplateOverload.fget_release_gil, W_CPPTemplateOverload.fset_release_gil),
+    __useffi__      = GetSetProperty(W_CPPTemplateOverload.fget_useffi,      W_CPPTemplateOverload.fset_useffi),
+    __doc__         = GetSetProperty(W_CPPTemplateOverload.fget_doc)
 )
 
 class W_CPPTemplateStaticOverload(W_CPPStaticOverload, TemplateOverloadMixin):
@@ -929,11 +1005,18 @@
 
 W_CPPTemplateStaticOverload.typedef = TypeDef(
     'CPPTemplateStaticOverload',
-    __get__      = interp2app(W_CPPTemplateStaticOverload.descr_get),
-    __getitem__  = interp2app(W_CPPTemplateStaticOverload.getitem),
-    __call__     = interp2app(W_CPPTemplateStaticOverload.call_args),
-    __useffi__   = GetSetProperty(W_CPPTemplateStaticOverload.fget_useffi, W_CPPTemplateStaticOverload.fset_useffi),
-    __doc__      = GetSetProperty(W_CPPTemplateStaticOverload.fget_doc)
+    __get__         = interp2app(W_CPPTemplateStaticOverload.descr_get),
+    __getitem__     = interp2app(W_CPPTemplateStaticOverload.getitem),
+    __call__        = interp2app(W_CPPTemplateStaticOverload.call_args),
+    __creates__     = GetSetProperty(W_CPPTemplateStaticOverload.fget_creates,
+                                     W_CPPTemplateStaticOverload.fset_creates),
+    __mempolicy__   = GetSetProperty(W_CPPTemplateStaticOverload.fget_mempolicy,
+                                     W_CPPTemplateStaticOverload.fset_mempolicy),
+    __release_gil__ = GetSetProperty(W_CPPTemplateStaticOverload.fget_release_gil,
+                                     W_CPPTemplateStaticOverload.fset_release_gil),
+    __useffi__      = GetSetProperty(W_CPPTemplateStaticOverload.fget_useffi,
+                                     W_CPPTemplateStaticOverload.fset_useffi),
+    __doc__         = GetSetProperty(W_CPPTemplateStaticOverload.fget_doc)
 )
 
 
diff --git a/pypy/module/_cppyy/test/pythonizables.cxx b/pypy/module/_cppyy/test/pythonizables.cxx
--- a/pypy/module/_cppyy/test/pythonizables.cxx
+++ b/pypy/module/_cppyy/test/pythonizables.cxx
@@ -27,3 +27,5 @@
 unsigned int pyzables::pass_mine_rp(Countable c) { return c.m_check; }
 unsigned int pyzables::pass_mine_rp_ref(const Countable& c) { return c.m_check; }
 unsigned int pyzables::pass_mine_rp_ptr(const Countable* c) { return c->m_check; }
+
+pyzables::Countable* pyzables::gime_naked_countable() { return new Countable{}; }
diff --git a/pypy/module/_cppyy/test/pythonizables.h b/pypy/module/_cppyy/test/pythonizables.h
--- a/pypy/module/_cppyy/test/pythonizables.h
+++ b/pypy/module/_cppyy/test/pythonizables.h
@@ -57,4 +57,6 @@
 unsigned int pass_mine_rp_ref(const Countable&);
 unsigned int pass_mine_rp_ptr(const Countable*);
 
+Countable* gime_naked_countable();
+
 } // namespace pyzables
diff --git a/pypy/module/_cppyy/test/test_advancedcpp.py b/pypy/module/_cppyy/test/test_advancedcpp.py
--- a/pypy/module/_cppyy/test/test_advancedcpp.py
+++ b/pypy/module/_cppyy/test/test_advancedcpp.py
@@ -686,11 +686,11 @@
         assert cppyy.gbl.my_global_double == 12.
         assert len(cppyy.gbl.my_global_array) == 500
         assert cppyy.gbl.my_global_string1 == "aap  noot  mies"
-        return     # next line currently crashes
         assert cppyy.gbl.my_global_string2 == "zus jet teun"
         # TODO: currently fails b/c double** not understood as &double*
         #assert cppyy.gbl.my_global_ptr[0] == 1234.
 
+        return
         v = cppyy.gbl.my_global_int_holders
         assert len(v) == 5
         expected_vals = [13, 42, 88, -1, 17]
diff --git a/pypy/module/_cppyy/test/test_pythonization.py b/pypy/module/_cppyy/test/test_pythonization.py
--- a/pypy/module/_cppyy/test/test_pythonization.py
+++ b/pypy/module/_cppyy/test/test_pythonization.py
@@ -17,14 +17,20 @@
         cls.w_datatypes = cls.space.appexec([], """():
             import ctypes, _cppyy
             _cppyy._post_import_startup()
+            class py(object):
+                pass
+            py.add_pythonization = _cppyy.add_pythonization
+            py.remove_pythonization = _cppyy.remove_pythonization
+            py.pin_type = _cppyy._pin_type
+            _cppyy.py = py
             return ctypes.CDLL(%r, ctypes.RTLD_GLOBAL)""" % (test_dct, ))
 
     def test00_api(self):
         """Test basic semantics of the pythonization API"""
 
-        import _cppyy
+        import _cppyy as cppyy
 
-        raises(TypeError, _cppyy.add_pythonization, 1)
+        raises(TypeError, cppyy.py.add_pythonization, 1)
 
         def pythonizor1(klass, name):
             pass
@@ -34,31 +40,26 @@
 
         pythonizor3 = pythonizor1
 
-        _cppyy.add_pythonization(pythonizor1)
-        assert _cppyy.remove_pythonization(pythonizor2) == False
-        assert _cppyy.remove_pythonization(pythonizor3) == True
-
-    def test01_more_api(self):
-        """Further API semantics"""
-
-        import _cppyy as cppyy
+        cppyy.py.add_pythonization(pythonizor1)
+        assert cppyy.py.remove_pythonization(pythonizor2) == False
+        assert cppyy.py.remove_pythonization(pythonizor3) == True
 
         def pythonizor(klass, name):
             if name == 'pyzables::SomeDummy1':
                 klass.test = 1
 
-        cppyy.add_pythonization(pythonizor)
+        cppyy.py.add_pythonization(pythonizor)
         assert cppyy.gbl.pyzables.SomeDummy1.test == 1
 
         def pythonizor(klass, name):
             if name == 'SomeDummy2':
                 klass.test = 2
-        cppyy.add_pythonization(pythonizor, 'pyzables')
+        cppyy.py.add_pythonization(pythonizor, 'pyzables')
 
         def pythonizor(klass, name):
             if name == 'pyzables::SomeDummy2':
                 klass.test = 3
-        cppyy.add_pythonization(pythonizor)
+        cppyy.py.add_pythonization(pythonizor)
 
         assert cppyy.gbl.pyzables.SomeDummy2.test == 2
 
@@ -66,28 +67,24 @@
             if name == 'TString':
                 klass.__len__ = klass.Length
 
-        cppyy.add_pythonization(root_pythonizor)
+        cppyy.py.add_pythonization(root_pythonizor)
 
         assert len(cppyy.gbl.TString("aap")) == 3
 
-    def test02_type_pinning(self):
+    def test01_type_pinning(self):
         """Verify pinnability of returns"""
 
         import _cppyy as cppyy
 
-        # TODO: disabled for now until decided on proper naming/iface
-        return
-
-        cppyy.gbl.pyzables.GimeDerived._creates = True
+        cppyy.gbl.pyzables.GimeDerived.__creates__ = True
 
         result = cppyy.gbl.pyzables.GimeDerived()
         assert type(result) == cppyy.gbl.pyzables.MyDerived
 
-        cppyy._pin_type(cppyy.gbl.pyzables.MyBase)
+        cppyy.py.pin_type(cppyy.gbl.pyzables.MyBase)
         assert type(result) == cppyy.gbl.pyzables.MyDerived
 
-
-    def test03_transparency(self):
+    def test02_transparency(self):
         """Transparent use of smart pointers"""
 
         import _cppyy as cppyy
@@ -101,7 +98,7 @@
         assert mine.__smartptr__().get().m_check == 0xcdcdcdcd
         assert mine.say_hi() == "Hi!"
 
-    def test04_converters(self):
+    def test03_converters(self):
         """Smart pointer argument passing"""
 
         import _cppyy as cppyy
@@ -126,7 +123,7 @@
         # cppyy.gbl.mine = mine
         pz.renew_mine()
 
-    def test05_executors(self):
+    def test04_executors(self):
         """Smart pointer return types"""
 
         import _cppyy as cppyy
@@ -154,3 +151,25 @@
         assert type(mine.__smartptr__()) == cppyy.gbl.std.shared_ptr(Countable)
         assert mine.__smartptr__().get().m_check == 0xcdcdcdcd
         assert mine.say_hi() == "Hi!"
+
+    def test05_creates_flag(self):
+        """Effect of creates flag on return type"""
+
+        import _cppyy as cppyy
+        import gc
+
+        pz = cppyy.gbl.pyzables
+        Countable = pz.Countable
+
+        gc.collect()
+        oldcount = Countable.sInstances     # there's eg. one global variable
+
+        pz.gime_naked_countable.__creates__ = True
+        for i in range(10):
+            cnt = pz.gime_naked_countable()
+            gc.collect()
+            assert Countable.sInstances == oldcount + 1
+        del cnt
+        gc.collect()
+
+        assert Countable.sInstances == oldcount
diff --git a/pypy/module/_cppyy/test/test_zjit.py b/pypy/module/_cppyy/test/test_zjit.py
--- a/pypy/module/_cppyy/test/test_zjit.py
+++ b/pypy/module/_cppyy/test/test_zjit.py
@@ -141,6 +141,9 @@
         self.w_TypeError           = FakeException(self, "TypeError")
         self.w_ValueError          = FakeException(self, "ValueError")
 
+        self.w_True                = FakeBool(True)
+        self.w_False               = FakeBool(False)
+
     def issequence_w(self, w_obj):
         return True
 

From pypy.commits at gmail.com  Tue Jul 17 02:01:38 2018
From: pypy.commits at gmail.com (wlav)
Date: Mon, 16 Jul 2018 23:01:38 -0700 (PDT)
Subject: [pypy-commit] pypy cppyy-packaging: improved dir() for namespaces
Message-ID: <5b4d8642.1c69fb81.da662.24d9@mx.google.com>

Author: Wim Lavrijsen <WLavrijsen at lbl.gov>
Branch: cppyy-packaging
Changeset: r94869:7fecec6b5a70
Date: 2018-07-15 21:43 -0700
http://bitbucket.org/pypy/pypy/changeset/7fecec6b5a70/

Log:	improved dir() for namespaces

diff --git a/pypy/module/_cppyy/pythonify.py b/pypy/module/_cppyy/pythonify.py
--- a/pypy/module/_cppyy/pythonify.py
+++ b/pypy/module/_cppyy/pythonify.py
@@ -29,7 +29,14 @@
 
 class CPPNamespaceMeta(CPPScopeMeta):
     def __dir__(self):
-        return self.__cppdecl__.__dir__()
+        # For Py3: can actually call base class __dir__ (lives in type)
+        values = set(self.__dict__.keys())
+        values.update(object.__dict__.keys())
+        values.update(type(self).__dict__.keys())
+
+        # add C++ entities
+        values.update(self.__cppdecl__.__dir__())
+        return list(values)
 
 class CPPClassMeta(CPPScopeMeta):
     pass

From pypy.commits at gmail.com  Tue Jul 17 02:01:40 2018
From: pypy.commits at gmail.com (wlav)
Date: Mon, 16 Jul 2018 23:01:40 -0700 (PDT)
Subject: [pypy-commit] pypy cppyy-packaging: remove unused variable
Message-ID: <5b4d8644.1c69fb81.c0dd4.3913@mx.google.com>

Author: Wim Lavrijsen <WLavrijsen at lbl.gov>
Branch: cppyy-packaging
Changeset: r94870:8e6c6830fc0c
Date: 2018-07-15 21:43 -0700
http://bitbucket.org/pypy/pypy/changeset/8e6c6830fc0c/

Log:	remove unused variable

diff --git a/pypy/module/_cppyy/converter.py b/pypy/module/_cppyy/converter.py
--- a/pypy/module/_cppyy/converter.py
+++ b/pypy/module/_cppyy/converter.py
@@ -115,7 +115,7 @@
         from pypy.module._cppyy.interp_cppyy import FastCallNotPossible
         raise FastCallNotPossible
 
-    def from_memory(self, space, w_obj, w_pycppclass, offset):
+    def from_memory(self, space, w_obj, offset):
         self._is_abstract(space)
 
     def to_memory(self, space, w_obj, w_value, offset):
@@ -142,7 +142,7 @@
         state = space.fromcache(ffitypes.State)
         return state.c_voidp
 
-    def from_memory(self, space, w_obj, w_pycppclass, offset):
+    def from_memory(self, space, w_obj, offset):
         # read access, so no copy needed
         address_value = self._get_raw_address(space, w_obj, offset)
         address = rffi.cast(rffi.ULONG, address_value)
@@ -184,7 +184,7 @@
         ba = rffi.cast(rffi.CCHARP, address)
         ba[capi.c_function_arg_typeoffset(space)] = 'o'
 
-    def from_memory(self, space, w_obj, w_pycppclass, offset):
+    def from_memory(self, space, w_obj, offset):
         # read access, so no copy needed
         address_value = self._get_raw_address(space, w_obj, offset)
         address = rffi.cast(rffi.ULONGP, address_value)
@@ -214,7 +214,7 @@
         x = rffi.cast(self.c_ptrtype, address)
         x[0] = self.default
 
-    def from_memory(self, space, w_obj, w_pycppclass, offset):
+    def from_memory(self, space, w_obj, offset):
         address = self._get_raw_address(space, w_obj, offset)
         rffiptr = rffi.cast(self.c_ptrtype, address)
         return self._wrap_object(space, rffiptr[0])
@@ -283,7 +283,7 @@
         x = rffi.cast(rffi.LONGP, address)
         x[0] = self._unwrap_object(space, w_obj)
 
-    def from_memory(self, space, w_obj, w_pycppclass, offset):
+    def from_memory(self, space, w_obj, offset):
         address = rffi.cast(rffi.CCHARP, self._get_raw_address(space, w_obj, offset))
         if address[0] == '\x01':
             return space.w_True
@@ -308,7 +308,7 @@
         x = rffi.cast(self.c_ptrtype, address)
         x[0] = self._unwrap_object(space, w_obj)
 
-    def from_memory(self, space, w_obj, w_pycppclass, offset):
+    def from_memory(self, space, w_obj, offset):
         address = rffi.cast(rffi.CCHARP, self._get_raw_address(space, w_obj, offset))
         return space.newbytes(address[0])
 
@@ -330,7 +330,7 @@
             fval = float(0.)
         self.default = r_singlefloat(fval)
 
-    def from_memory(self, space, w_obj, w_pycppclass, offset):
+    def from_memory(self, space, w_obj, offset):
         address = self._get_raw_address(space, w_obj, offset)
         rffiptr = rffi.cast(self.c_ptrtype, address)
         return self._wrap_object(space, rffiptr[0])
@@ -383,7 +383,7 @@
         ba = rffi.cast(rffi.CCHARP, address)
         ba[capi.c_function_arg_typeoffset(space)] = 'p'
 
-    def from_memory(self, space, w_obj, w_pycppclass, offset):
+    def from_memory(self, space, w_obj, offset):
         address = self._get_raw_address(space, w_obj, offset)
         charpptr = rffi.cast(rffi.CCHARPP, address)
         return space.newtext(rffi.charp2str(charpptr[0]))
@@ -397,7 +397,7 @@
     def __init__(self, space, extra):
         self.size = extra
 
-    def from_memory(self, space, w_obj, w_pycppclass, offset):
+    def from_memory(self, space, w_obj, offset):
         address = self._get_raw_address(space, w_obj, offset)
         charpptr = rffi.cast(rffi.CCHARP, address)
         if 0 <= self.size and self.size != 2**31-1:   # cling's code for "unknown" (?)
@@ -430,7 +430,7 @@
         x = rffi.cast(rffi.VOIDPP, address)
         x[0] = self._unwrap_object(space, w_obj)
 
-    def from_memory(self, space, w_obj, w_pycppclass, offset):
+    def from_memory(self, space, w_obj, offset):
         # returned as a long value for the address (INTPTR_T is not proper
         # per se, but rffi does not come with a PTRDIFF_T)
         address = self._get_raw_address(space, w_obj, offset)
@@ -542,7 +542,7 @@
         from pypy.module._cppyy.interp_cppyy import FastCallNotPossible
         raise FastCallNotPossible       # TODO: by-value is a jit_libffi special case
 
-    def from_memory(self, space, w_obj, w_pycppclass, offset):
+    def from_memory(self, space, w_obj, offset):
         address = rffi.cast(capi.C_OBJECT, self._get_raw_address(space, w_obj, offset))
         from pypy.module._cppyy import interp_cppyy
         return interp_cppyy.wrap_cppinstance(space, address, self.clsdecl, do_cast=False)
@@ -562,7 +562,7 @@
                 return capi.C_NULL_OBJECT
             raise e
 
-    def from_memory(self, space, w_obj, w_pycppclass, offset):
+    def from_memory(self, space, w_obj, offset):
         address = rffi.cast(capi.C_OBJECT, self._get_raw_address(space, w_obj, offset))
         from pypy.module._cppyy import interp_cppyy
         return interp_cppyy.wrap_cppinstance(space, address, self.clsdecl, do_cast=False)
@@ -588,7 +588,7 @@
         from pypy.module._cppyy.interp_cppyy import FastCallNotPossible
         raise FastCallNotPossible
 
-    def from_memory(self, space, w_obj, w_pycppclass, offset):
+    def from_memory(self, space, w_obj, offset):
         address = rffi.cast(capi.C_OBJECT, self._get_raw_address(space, w_obj, offset))
         from pypy.module._cppyy import interp_cppyy
         return interp_cppyy.wrap_cppinstance(
@@ -792,7 +792,7 @@
         ba = rffi.cast(rffi.CCHARP, address)
         ba[capi.c_function_arg_typeoffset(space)] = self.typecode
 
-    def from_memory(self, space, w_obj, w_pycppclass, offset):
+    def from_memory(self, space, w_obj, offset):
         address = rffi.cast(capi.C_OBJECT, self._get_raw_address(space, w_obj, offset))
         from pypy.module._cppyy import interp_cppyy
         return interp_cppyy.wrap_cppinstance(space, address,
@@ -801,7 +801,7 @@
 class SmartPtrPtrConverter(SmartPtrConverter):
     typecode    = 'o'
 
-    def from_memory(self, space, w_obj, w_pycppclass, offset):
+    def from_memory(self, space, w_obj, offset):
         self._is_abstract(space)
 
     def to_memory(self, space, w_obj, w_value, offset):
@@ -813,7 +813,7 @@
 
 
 class MacroConverter(TypeConverter):
-    def from_memory(self, space, w_obj, w_pycppclass, offset):
+    def from_memory(self, space, w_obj, offset):
         # TODO: get the actual type info from somewhere ...
         address = self._get_raw_address(space, w_obj, offset)
         longptr = rffi.cast(rffi.LONGP, address)
diff --git a/pypy/module/_cppyy/interp_cppyy.py b/pypy/module/_cppyy/interp_cppyy.py
--- a/pypy/module/_cppyy/interp_cppyy.py
+++ b/pypy/module/_cppyy/interp_cppyy.py
@@ -1057,7 +1057,7 @@
             raise oefmt(self.space.w_AttributeError,
                         "attribute access requires an instance")
         offset = self._get_offset(cppinstance)
-        return self.converter.from_memory(self.space, w_cppinstance, w_pycppclass, offset)
+        return self.converter.from_memory(self.space, w_cppinstance, offset)
 
     def set(self, w_cppinstance, w_value):
         cppinstance = self.space.interp_w(W_CPPInstance, w_cppinstance, can_be_None=True)
@@ -1094,7 +1094,7 @@
         return self.offset
 
     def get(self, w_cppinstance, w_pycppclass):
-        return self.converter.from_memory(self.space, self.space.w_None, w_pycppclass, self.offset)
+        return self.converter.from_memory(self.space, self.space.w_None, self.offset)
 
     def set(self, w_cppinstance, w_value):
         self.converter.to_memory(self.space, self.space.w_None, w_value, self.offset)

From pypy.commits at gmail.com  Tue Jul 17 02:01:42 2018
From: pypy.commits at gmail.com (wlav)
Date: Mon, 16 Jul 2018 23:01:42 -0700 (PDT)
Subject: [pypy-commit] pypy cppyy-packaging: store free functions on
 namespace dict, not namespace's class dict
Message-ID: <5b4d8646.1c69fb81.9a489.182b@mx.google.com>

Author: Wim Lavrijsen <WLavrijsen at lbl.gov>
Branch: cppyy-packaging
Changeset: r94871:ff8881eb57bb
Date: 2018-07-16 13:02 -0700
http://bitbucket.org/pypy/pypy/changeset/ff8881eb57bb/

Log:	store free functions on namespace dict, not namespace's class dict

diff --git a/pypy/module/_cppyy/pythonify.py b/pypy/module/_cppyy/pythonify.py
--- a/pypy/module/_cppyy/pythonify.py
+++ b/pypy/module/_cppyy/pythonify.py
@@ -317,7 +317,7 @@
     if not cppitem:
         try:
             cppitem = scope.__cppdecl__.get_overload(name)
-            setattr(scope.__class__, name, cppitem)
+            setattr(scope, name, cppitem)
             pycppitem = getattr(scope, name)      # binds function as needed
         except AttributeError:
             pass

From pypy.commits at gmail.com  Tue Jul 17 02:01:44 2018
From: pypy.commits at gmail.com (wlav)
Date: Mon, 16 Jul 2018 23:01:44 -0700 (PDT)
Subject: [pypy-commit] pypy cppyy-packaging: make sure to pass w_this for
 non-templated overloads of templates
Message-ID: <5b4d8648.1c69fb81.33845.0ff7@mx.google.com>

Author: Wim Lavrijsen <WLavrijsen at lbl.gov>
Branch: cppyy-packaging
Changeset: r94872:1bad6c9de6b7
Date: 2018-07-16 22:41 -0700
http://bitbucket.org/pypy/pypy/changeset/1bad6c9de6b7/

Log:	make sure to pass w_this for non-templated overloads of templates

diff --git a/pypy/module/_cppyy/interp_cppyy.py b/pypy/module/_cppyy/interp_cppyy.py
--- a/pypy/module/_cppyy/interp_cppyy.py
+++ b/pypy/module/_cppyy/interp_cppyy.py
@@ -926,7 +926,7 @@
 
         # try existing overloads or compile-time instantiations
         try:
-            return W_CPPOverload.call_args(self, args_w)
+            return W_CPPOverload.call_args(self, [self.w_this]+args_w)
         except Exception:
             pass
 

From pypy.commits at gmail.com  Tue Jul 17 07:32:24 2018
From: pypy.commits at gmail.com (arigo)
Date: Tue, 17 Jul 2018 04:32:24 -0700 (PDT)
Subject: [pypy-commit] pypy default: update to cffi/b3b6b86cb30c
Message-ID: <5b4dd3c8.1c69fb81.7e11.7e74@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r94873:94d4f08d6056
Date: 2018-07-17 12:32 +0200
http://bitbucket.org/pypy/pypy/changeset/94d4f08d6056/

Log:	update to cffi/b3b6b86cb30c

diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -3935,8 +3935,8 @@
 
 def test_char_pointer_conversion():
     import warnings
-    assert __version__.startswith(("1.8", "1.9", "1.10", "1.11", "1.12")), (
-        "consider turning the warning into an error")
+    assert __version__.startswith("1."), (
+        "the warning will be an error if we ever release cffi 2.x")
     BCharP = new_pointer_type(new_primitive_type("char"))
     BIntP = new_pointer_type(new_primitive_type("int"))
     BVoidP = new_pointer_type(new_void_type())

From pypy.commits at gmail.com  Wed Jul 18 06:04:21 2018
From: pypy.commits at gmail.com (arigo)
Date: Wed, 18 Jul 2018 03:04:21 -0700 (PDT)
Subject: [pypy-commit] pypy default: rvmprof.dummy is not really working,
 at least for pypy.  Try to fix it some more, 
Message-ID: <5b4f10a5.1c69fb81.eda8f.ab97@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r94874:0ee5333ce97e
Date: 2018-07-18 10:46 +0200
http://bitbucket.org/pypy/pypy/changeset/0ee5333ce97e/

Log:	rvmprof.dummy is not really working, at least for pypy. Try to fix
	it some more, but give up for now. Instead, robustly detect that
	vmprof is not supported from pypyoption and disable the '_vmprof'
	and 'faulthandler' modules.

diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py
--- a/pypy/config/pypyoption.py
+++ b/pypy/config/pypyoption.py
@@ -39,14 +39,10 @@
     "_csv", "_cppyy", "_pypyjson", "_jitlog"
 ])
 
-from rpython.jit.backend import detect_cpu
-try:
-    if detect_cpu.autodetect().startswith('x86'):
-        if not sys.platform.startswith('openbsd'):
-            working_modules.add('_vmprof')
-            working_modules.add('faulthandler')
-except detect_cpu.ProcessorAutodetectError:
-    pass
+import rpython.rlib.rvmprof.cintf
+if rpython.rlib.rvmprof.cintf.IS_SUPPORTED:
+    working_modules.add('_vmprof')
+    working_modules.add('faulthandler')
 
 translation_modules = default_modules.copy()
 translation_modules.update([
@@ -318,3 +314,4 @@
     parser = to_optparse(config) #, useoptions=["translation.*"])
     option, args = parser.parse_args()
     print config
+    print working_modules
diff --git a/rpython/rlib/rvmprof/dummy.py b/rpython/rlib/rvmprof/dummy.py
--- a/rpython/rlib/rvmprof/dummy.py
+++ b/rpython/rlib/rvmprof/dummy.py
@@ -1,6 +1,7 @@
 from rpython.rlib.objectmodel import specialize
 
 class DummyVMProf(object):
+    is_enabled = False
 
     def __init__(self):
         self._unique_id = 0
diff --git a/rpython/rlib/rvmprof/rvmprof.py b/rpython/rlib/rvmprof/rvmprof.py
--- a/rpython/rlib/rvmprof/rvmprof.py
+++ b/rpython/rlib/rvmprof/rvmprof.py
@@ -23,6 +23,7 @@
 VMPROF_GC_TAG = 5
 
 class VMProfError(Exception):
+    msg = ''   # annotation hack
     def __init__(self, msg):
         self.msg = msg
     def __str__(self):
diff --git a/rpython/rlib/rvmprof/traceback.py b/rpython/rlib/rvmprof/traceback.py
--- a/rpython/rlib/rvmprof/traceback.py
+++ b/rpython/rlib/rvmprof/traceback.py
@@ -13,6 +13,8 @@
     array_length).  The caller must free array_p.  Not for signal handlers:
     for these, call vmprof_get_traceback() from C code.
     """
+    if not cintf.IS_SUPPORTED:
+        return (None, 0)
     _cintf = rvmprof._get_vmprof().cintf
     size = estimate_number_of_entries * 2 + 4
     stack = cintf.get_rvmprof_stack()
@@ -47,6 +49,8 @@
     'code_obj' may be None if it can't be determined.  'loc' is one
     of the LOC_xxx constants.
     """
+    if not cintf.IS_SUPPORTED:
+        return
     i = 0
     while i < array_length - 1:
         tag = array_p[i]

From pypy.commits at gmail.com  Wed Jul 18 10:41:43 2018
From: pypy.commits at gmail.com (arigo)
Date: Wed, 18 Jul 2018 07:41:43 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Start to debug 'multiprocessing' on win32,
 but found out that '_winapi.py' is not complete at all.
Message-ID: <5b4f51a7.1c69fb81.21c44.f106@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r94875:cad71ec84cef
Date: 2018-07-18 16:38 +0200
http://bitbucket.org/pypy/pypy/changeset/cad71ec84cef/

Log:	Start to debug 'multiprocessing' on win32, but found out that
	'_winapi.py' is not complete at all.

diff --git a/lib_pypy/_winapi.py b/lib_pypy/_winapi.py
--- a/lib_pypy/_winapi.py
+++ b/lib_pypy/_winapi.py
@@ -98,9 +98,11 @@
     def GetOverlappedResult(self, wait):
         transferred = _ffi.new('DWORD[1]', [0])
         res = _kernel32.GetOverlappedResult(self.handle, self.overlapped, transferred, wait != 0)
-        if not res:
-            res = GetLastError()
-        if res in (ERROR_SUCCESS, ERROR_MORE_DATA, ERROR_OPERATION_ABORTED):
+        if res:
+            err = ERROR_SUCCESS
+        else:
+            err = GetLastError()
+        if err in (ERROR_SUCCESS, ERROR_MORE_DATA, ERROR_OPERATION_ABORTED):
             self.completed = 1
             self.pending = 0
         elif res == ERROR_IO_INCOMPLETE:
@@ -133,7 +135,7 @@
         assert success == 0
         err = _kernel32.GetLastError()
         if err == ERROR_IO_PENDING:
-            overlapped[0].pending = 1
+            ov.pending = 1
         elif err == ERROR_PIPE_CONNECTED:
             _kernel32.SetEvent(ov.overlapped[0].hEvent)
         else:

From pypy.commits at gmail.com  Wed Jul 18 16:49:32 2018
From: pypy.commits at gmail.com (arigo)
Date: Wed, 18 Jul 2018 13:49:32 -0700 (PDT)
Subject: [pypy-commit] pypy default: Apply patch to have vmprof work on
 FreeBSD
Message-ID: <5b4fa7dc.1c69fb81.81b73.6424@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r94876:587bebd53960
Date: 2018-07-18 22:47 +0200
http://bitbucket.org/pypy/pypy/changeset/587bebd53960/

Log:	Apply patch to have vmprof work on FreeBSD (thanks David C. on pypy-
	dev)

diff --git a/rpython/rlib/rvmprof/cintf.py b/rpython/rlib/rvmprof/cintf.py
--- a/rpython/rlib/rvmprof/cintf.py
+++ b/rpython/rlib/rvmprof/cintf.py
@@ -17,7 +17,7 @@
 
 # vmprof works only on x86 for now
 IS_SUPPORTED = False
-if sys.platform in ('darwin', 'linux', 'linux2'):
+if sys.platform in ('darwin', 'linux', 'linux2', 'freebsd10', 'freebsd11'):
     try:
         IS_SUPPORTED = detect_cpu.autodetect().startswith('x86')
     except detect_cpu.ProcessorAutodetectError:
diff --git a/rpython/rlib/rvmprof/src/shared/vmprof_unix.h b/rpython/rlib/rvmprof/src/shared/vmprof_unix.h
--- a/rpython/rlib/rvmprof/src/shared/vmprof_unix.h
+++ b/rpython/rlib/rvmprof/src/shared/vmprof_unix.h
@@ -24,6 +24,9 @@
 
 #include "vmprof_mt.h"
 
+#ifdef __FreeBSD__
+#include <ucontext.h>
+#endif
 #include <signal.h>
 
 RPY_EXTERN void vmprof_ignore_signals(int ignored);

From pypy.commits at gmail.com  Thu Jul 19 00:42:11 2018
From: pypy.commits at gmail.com (wlav)
Date: Wed, 18 Jul 2018 21:42:11 -0700 (PDT)
Subject: [pypy-commit] pypy cppyy-packaging: vector<bool>'s base class has
 no 'data'
Message-ID: <5b5016a3.1c69fb81.eae6.b1d8@mx.google.com>

Author: Wim Lavrijsen <WLavrijsen at lbl.gov>
Branch: cppyy-packaging
Changeset: r94877:e3f75a873428
Date: 2018-07-18 21:20 -0700
http://bitbucket.org/pypy/pypy/changeset/e3f75a873428/

Log:	vector<bool>'s base class has no 'data'

diff --git a/pypy/module/_cppyy/pythonify.py b/pypy/module/_cppyy/pythonify.py
--- a/pypy/module/_cppyy/pythonify.py
+++ b/pypy/module/_cppyy/pythonify.py
@@ -459,12 +459,13 @@
         pyclass.__init__ = vector_init
 
         # size-up the return of data()
-        pyclass.__real_data = pyclass.data
-        def data_with_len(self):
-            arr = self.__real_data()
-            arr.reshape((len(self),))
-            return arr
-        pyclass.data = data_with_len
+        if hasattr(pyclass, 'data'):   # not the case for e.g. vector<bool>
+            pyclass.__real_data = pyclass.data
+            def data_with_len(self):
+                arr = self.__real_data()
+                arr.reshape((len(self),))
+                return arr
+            pyclass.data = data_with_len
 
     # combine __getitem__ and __len__ to make a pythonized __getitem__
     if '__getitem__' in pyclass.__dict__ and '__len__' in pyclass.__dict__:

From pypy.commits at gmail.com  Thu Jul 19 00:42:43 2018
From: pypy.commits at gmail.com (wlav)
Date: Wed, 18 Jul 2018 21:42:43 -0700 (PDT)
Subject: [pypy-commit] pypy cppyy-packaging: converter for builtin arrays of
 instances
Message-ID: <5b5016c3.1c69fb81.8882.4598@mx.google.com>

Author: Wim Lavrijsen <WLavrijsen at lbl.gov>
Branch: cppyy-packaging
Changeset: r94878:046b87bc288f
Date: 2018-07-18 21:21 -0700
http://bitbucket.org/pypy/pypy/changeset/046b87bc288f/

Log:	converter for builtin arrays of instances

diff --git a/pypy/module/_cppyy/converter.py b/pypy/module/_cppyy/converter.py
--- a/pypy/module/_cppyy/converter.py
+++ b/pypy/module/_cppyy/converter.py
@@ -622,6 +622,24 @@
             lltype.free(self.ref_buffer, flavor='raw')
             self.ref_buffer = lltype.nullptr(rffi.VOIDPP.TO)
 
+class InstanceArrayConverter(InstancePtrConverter):
+    _immutable_fields_ = ['size']
+
+    def __init__(self, space, clsdecl, array_size):
+        InstancePtrConverter.__init__(self, space, clsdecl)
+        if array_size <= 0:
+            self.size = sys.maxint
+        else:
+            self.size = array_size
+
+    def from_memory(self, space, w_obj, offset):
+        address = rffi.cast(capi.C_OBJECT, self._get_raw_address(space, w_obj, offset))
+        return lowlevelviews.W_ArrayOfInstances(space, self.clsdecl, address, self.size)
+
+    def to_memory(self, space, w_obj, w_value, offset):
+        self._is_abstract(space)
+
+
 class StdStringConverter(InstanceConverter):
     def __init__(self, space, extra):
         from pypy.module._cppyy import interp_cppyy
@@ -846,20 +864,19 @@
         pass
 
     # match of decorated, unqualified type
-    compound = helper.compound(name)
+    cpd = helper.compound(name)
     clean_name = capi.c_resolve_name(space, helper.clean_type(name))
     try:
-        return _converters[clean_name+compound](space, default)
+        return _converters[clean_name+cpd](space, default)
     except KeyError:
         pass
 
-    # arrays
+    # arrays (array_size may be negative, meaning: no size or no size found)
+    array_size = helper.array_size(_name)     # uses original arg
     try:
-        # array_index may be negative to indicate no size or no size found
-        array_size = helper.array_size(_name)     # uses original arg
         # TODO: using clean_name here drops const (e.g. const char[] will
         # never be seen this way)
-        return _a_converters[clean_name+compound](space, array_size)
+        return _a_converters[clean_name+cpd](space, array_size)
     except KeyError:
         pass
 
@@ -873,24 +890,27 @@
         # check smart pointer type
         check_smart = capi.c_smartptr_info(space, clean_name)
         if check_smart[0]:
-            if compound == '':
+            if cpd == '':
                 return SmartPtrConverter(space, clsdecl, check_smart[1], check_smart[2])
-            elif compound == '*':
+            elif cpd == '*':
                 return SmartPtrPtrConverter(space, clsdecl, check_smart[1], check_smart[2])
-            elif compound == '&':
+            elif cpd == '&':
                 return SmartPtrRefConverter(space, clsdecl, check_smart[1], check_smart[2])
             # fall through: can still return smart pointer in non-smart way
 
         # type check for the benefit of the annotator
-        if compound == "*":
+        if cpd == "*":
             return InstancePtrConverter(space, clsdecl)
-        elif compound == "&":
+        elif cpd == "&":
             return InstanceRefConverter(space, clsdecl)
-        elif compound == "&&":
+        elif cpd == "&&":
             return InstanceMoveConverter(space, clsdecl)
-        elif compound == "**":
+        elif cpd in ["**", "*[]", "&*"]:
             return InstancePtrPtrConverter(space, clsdecl)
-        elif compound == "":
+        elif cpd == "[]" and array_size > 0:
+            # TODO: retrieve dimensions
+            return InstanceArrayConverter(space, clsdecl, array_size)
+        elif cpd == "":
             return InstanceConverter(space, clsdecl)
     elif "(anonymous)" in name:
         # special case: enum w/o a type name
@@ -902,7 +922,7 @@
             return FunctionPointerConverter(space, name[pos+2:])
 
     # void* or void converter (which fails on use)
-    if 0 <= compound.find('*'):
+    if 0 <= cpd.find('*'):
         return VoidPtrConverter(space, default)  # "user knows best"
 
     # return a void converter here, so that the class can be build even
diff --git a/pypy/module/_cppyy/interp_cppyy.py b/pypy/module/_cppyy/interp_cppyy.py
--- a/pypy/module/_cppyy/interp_cppyy.py
+++ b/pypy/module/_cppyy/interp_cppyy.py
@@ -1040,7 +1040,7 @@
         self.space = space
         self.scope = decl_scope
         self.converter = converter.get_converter(self.space, type_name, '')
-        self.offset = offset
+        self.offset = rffi.cast(rffi.LONG, offset)
 
     def _get_offset(self, cppinstance):
         if cppinstance:
diff --git a/pypy/module/_cppyy/lowlevelviews.py b/pypy/module/_cppyy/lowlevelviews.py
--- a/pypy/module/_cppyy/lowlevelviews.py
+++ b/pypy/module/_cppyy/lowlevelviews.py
@@ -3,10 +3,16 @@
 # a few more methods allowing such information to be set. Afterwards, it is
 # simple to pass these views on to e.g. numpy (w/o the need to copy).
 
-from pypy.interpreter.error import oefmt
+from pypy.interpreter.error import OperationError, oefmt
 from pypy.interpreter.gateway import interp2app, unwrap_spec
 from pypy.interpreter.typedef import TypeDef, GetSetProperty, interp_attrproperty_w
+from pypy.interpreter.baseobjspace import W_Root
+
+from rpython.rtyper.lltypesystem import rffi
+
 from pypy.module._rawffi.array import W_ArrayInstance
+from pypy.module._rawffi.interp_rawffi import segfault_exception
+from pypy.module._cppyy import capi
 
 
 class W_LowLevelView(W_ArrayInstance):
@@ -52,3 +58,33 @@
 )
 W_ArrayInstance.typedef.acceptable_as_base_class = False
 
+
+class W_ArrayOfInstances(W_Root):
+    _attrs_ = ['converter', 'baseaddress', 'clssize', 'length']
+    _immutable_fields_ = ['converter', 'baseaddress', 'clssize']
+
+    def __init__(self, space, clsdecl, address, length):
+        from pypy.module._cppyy import converter
+        self.converter   = converter.get_converter(space, clsdecl.name, '')
+        self.baseaddress = address
+        self.clssize     = capi.c_size_of_klass(space, clsdecl)
+        self.length      = length
+
+    @unwrap_spec(idx=int)
+    def getitem(self, space, idx):
+        if not self.baseaddress:
+            raise segfault_exception(space, "accessing elements of freed array")
+        if idx >= self.length or idx < 0:
+            raise OperationError(space.w_IndexError, space.w_None)
+        itemaddress = rffi.cast(rffi.LONG, self.baseaddress+idx*self.clssize)
+        return self.converter.from_memory(space, space.w_None, itemaddress)
+
+    def getlength(self, space):
+        return space.newint(self.length)
+
+W_ArrayOfInstances.typedef = TypeDef(
+    'ArrayOfInstances',
+    __getitem__ = interp2app(W_ArrayOfInstances.getitem),
+    __len__     = interp2app(W_ArrayOfInstances.getlength),
+)
+W_ArrayOfInstances.typedef.acceptable_as_base_class = False
diff --git a/pypy/module/_cppyy/test/test_advancedcpp.py b/pypy/module/_cppyy/test/test_advancedcpp.py
--- a/pypy/module/_cppyy/test/test_advancedcpp.py
+++ b/pypy/module/_cppyy/test/test_advancedcpp.py
@@ -690,7 +690,6 @@
         # TODO: currently fails b/c double** not understood as &double*
         #assert cppyy.gbl.my_global_ptr[0] == 1234.
 
-        return
         v = cppyy.gbl.my_global_int_holders
         assert len(v) == 5
         expected_vals = [13, 42, 88, -1, 17]

From pypy.commits at gmail.com  Thu Jul 19 14:04:07 2018
From: pypy.commits at gmail.com (wlav)
Date: Thu, 19 Jul 2018 11:04:07 -0700 (PDT)
Subject: [pypy-commit] pypy cppyy-packaging: support for vector<bool>
Message-ID: <5b50d297.1c69fb81.8c37a.189e@mx.google.com>

Author: Wim Lavrijsen <WLavrijsen at lbl.gov>
Branch: cppyy-packaging
Changeset: r94879:e3fd3632d44e
Date: 2018-07-19 10:43 -0700
http://bitbucket.org/pypy/pypy/changeset/e3fd3632d44e/

Log:	support for vector<bool>

diff --git a/pypy/module/_cppyy/capi/loadable_capi.py b/pypy/module/_cppyy/capi/loadable_capi.py
--- a/pypy/module/_cppyy/capi/loadable_capi.py
+++ b/pypy/module/_cppyy/capi/loadable_capi.py
@@ -274,7 +274,8 @@
 
             'stdvector_valuetype'      : ([c_ccharp],                 c_ccharp),
             'stdvector_valuesize'      : ([c_ccharp],                 c_size_t),
-
+            'vectorbool_getitem'       : ([c_object, c_int],          c_int),
+            'vectorbool_setitem'       : ([c_object, c_int, c_int],   c_void),
         }
 
         # size/offset are backend-specific but fixed after load
@@ -657,17 +658,43 @@
     return charp2str_free(space, call_capi(space, 'stdvector_valuetype', [_ArgS(pystr)]))
 def c_stdvector_valuesize(space, pystr):
     return _cdata_to_size_t(space, call_capi(space, 'stdvector_valuesize', [_ArgS(pystr)]))
+def c_vectorbool_getitem(space, vbool, idx):
+    return call_capi(space, 'vectorbool_getitem', [_ArgH(vbool), _ArgL(idx)])
+def c_vectorbool_setitem(space, vbool, idx, value):
+    call_capi(space, 'vectorbool_setitem', [_ArgH(vbool), _ArgL(idx), _ArgL(value)])
 
 
 # TODO: factor these out ...
 # pythonizations
 def stdstring_c_str(space, w_self):
     """Return a python string taking into account \0"""
-
     from pypy.module._cppyy import interp_cppyy
     cppstr = space.interp_w(interp_cppyy.W_CPPInstance, w_self, can_be_None=False)
     return space.newtext(c_stdstring2charp(space, cppstr._rawobject))
 
+def vbool_getindex(space, w_vbool, w_idx):
+    idx = space.getindex_w(w_idx, space.w_IndexError, "std::vector<bool> index")
+    sz = space.len_w(w_vbool)
+    if idx < 0: idx += sz
+    if idx < 0 or idx >= sz:
+        raise IndexError
+    return idx
+
+def vectorbool_getitem(space, w_self, w_idx):
+    """Index a std::vector<bool>, return the value"""
+    from pypy.module._cppyy import interp_cppyy
+    vbool = space.interp_w(interp_cppyy.W_CPPInstance, w_self, can_be_None=False)
+    idx = vbool_getindex(space, w_self, w_idx)
+    item = c_vectorbool_getitem(space, vbool._rawobject, idx)
+    return space.newbool(space.is_true(item))
+
+def vectorbool_setitem(space, w_self, w_idx, w_value):
+    """Index a std::vector<bool>, set the value"""
+    from pypy.module._cppyy import interp_cppyy
+    vbool = space.interp_w(interp_cppyy.W_CPPInstance, w_self, can_be_None=False)
+    idx = vbool_getindex(space, w_self, w_idx)
+    c_vectorbool_setitem(space, vbool._rawobject, idx, int(space.is_true(w_value)))
+
 # setup pythonizations for later use at run-time
 _pythonizations = {}
 def register_pythonizations(space):
@@ -678,6 +705,9 @@
         ### std::string
         stdstring_c_str,
 
+        ### std::vector<bool>
+        vectorbool_getitem,
+        vectorbool_setitem,
     ]
 
     for f in allfuncs:
@@ -692,3 +722,7 @@
         space.setattr(w_pycppclass, space.newtext("c_str"), _pythonizations["stdstring_c_str"])
         _method_alias(space, w_pycppclass, "_cppyy_as_builtin", "c_str")
         _method_alias(space, w_pycppclass, "__str__",           "c_str")
+
+    if name == "std::vector<bool>":
+        space.setattr(w_pycppclass, space.newtext("__getitem__"), _pythonizations["vectorbool_getitem"])
+        space.setattr(w_pycppclass, space.newtext("__setitem__"), _pythonizations["vectorbool_setitem"])
diff --git a/pypy/module/_cppyy/include/capi.h b/pypy/module/_cppyy/include/capi.h
--- a/pypy/module/_cppyy/include/capi.h
+++ b/pypy/module/_cppyy/include/capi.h
@@ -219,6 +219,10 @@
     const char* cppyy_stdvector_valuetype(const char* clname);
     RPY_EXTERN
     size_t      cppyy_stdvector_valuesize(const char* clname);
+    RPY_EXTERN
+    int         cppyy_vectorbool_getitem(cppyy_object_t ptr, int idx);
+    RPY_EXTERN
+    void        cppyy_vectorbool_setitem(cppyy_object_t ptr, int idx, int value);
 
 #ifdef __cplusplus
 }
diff --git a/pypy/module/_cppyy/pythonify.py b/pypy/module/_cppyy/pythonify.py
--- a/pypy/module/_cppyy/pythonify.py
+++ b/pypy/module/_cppyy/pythonify.py
@@ -431,18 +431,19 @@
     # not on vector, which is pythonized in the capi (interp-level; there is
     # also the fallback on the indexed __getitem__, but that is slower)
 # TODO:    if not (0 <= name.find('vector') <= 5):
-    if ('begin' in pyclass.__dict__ and 'end' in pyclass.__dict__):
-        if _cppyy._scope_byname(name+'::iterator') or \
-                _cppyy._scope_byname(name+'::const_iterator'):
-            def __iter__(self):
-                i = self.begin()
-                while i != self.end():
-                    yield i.__deref__()
-                    i.__preinc__()
-                i.__destruct__()
-                raise StopIteration
-            pyclass.__iter__ = __iter__
-        # else: rely on numbered iteration
+    if not (0 <= name.find('vector<bool') <= 5):
+        if ('begin' in pyclass.__dict__ and 'end' in pyclass.__dict__):
+            if _cppyy._scope_byname(name+'::iterator') or \
+                    _cppyy._scope_byname(name+'::const_iterator'):
+                def __iter__(self):
+                    i = self.begin()
+                    while i != self.end():
+                        yield i.__deref__()
+                        i.__preinc__()
+                    i.__destruct__()
+                    raise StopIteration
+                pyclass.__iter__ = __iter__
+            # else: rely on numbered iteration
 
     # add python collection based initializer
     if 0 <= name.find('vector') <= 5:
diff --git a/pypy/module/_cppyy/test/test_stltypes.py b/pypy/module/_cppyy/test/test_stltypes.py
--- a/pypy/module/_cppyy/test/test_stltypes.py
+++ b/pypy/module/_cppyy/test/test_stltypes.py
@@ -197,6 +197,26 @@
         assert v2[-1] == v[-2]
         assert v2[self.N-4] == v[-2]
 
+    def test07_vector_bool(self):
+        """Usability of std::vector<bool> which can be a specialization"""
+
+        import _cppyy as cppyy
+
+        vb = cppyy.gbl.std.vector(bool)(8)
+        assert [x for x in vb] == [False]*8
+
+        vb[0] = True
+        assert vb[0]
+        vb[-1] = True
+        assert vb[7]
+
+        print [x for x in vb]
+
+        assert [x for x in vb] == [True]+[False]*6+[True]
+
+        assert len(vb[4:8]) == 4
+        assert list(vb[4:8]) == [False]*3+[True]
+
 
 class AppTestSTLSTRING:
     spaceconfig = dict(usemodules=['_cppyy', '_rawffi', 'itertools'])

From pypy.commits at gmail.com  Fri Jul 20 03:27:54 2018
From: pypy.commits at gmail.com (arigo)
Date: Fri, 20 Jul 2018 00:27:54 -0700 (PDT)
Subject: [pypy-commit] pypy default: Issue #2848
Message-ID: <5b518efa.1c69fb81.f2f93.454a@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r94880:2ca5d2f3a174
Date: 2018-07-20 09:25 +0200
http://bitbucket.org/pypy/pypy/changeset/2ca5d2f3a174/

Log:	Issue #2848

	Patch for GNU Hurd.

diff --git a/pypy/module/_multiprocessing/interp_semaphore.py b/pypy/module/_multiprocessing/interp_semaphore.py
--- a/pypy/module/_multiprocessing/interp_semaphore.py
+++ b/pypy/module/_multiprocessing/interp_semaphore.py
@@ -57,7 +57,7 @@
         TIMESPEC = platform.Struct('struct timespec', [('tv_sec', rffi.TIME_T),
                                                        ('tv_nsec', rffi.LONG)])
         SEM_FAILED = platform.ConstantInteger('SEM_FAILED')
-        SEM_VALUE_MAX = platform.ConstantInteger('SEM_VALUE_MAX')
+        SEM_VALUE_MAX = platform.DefinedConstantInteger('SEM_VALUE_MAX')
         SEM_TIMED_WAIT = platform.Has('sem_timedwait')
         SEM_T_SIZE = platform.SizeOf('sem_t')
 
@@ -70,6 +70,8 @@
     #                rffi.cast(SEM_T, config['SEM_FAILED'])
     SEM_FAILED     = config['SEM_FAILED']
     SEM_VALUE_MAX  = config['SEM_VALUE_MAX']
+    if SEM_VALUE_MAX is None:      # on Hurd
+        SEM_VALUE_MAX = sys.maxint
     SEM_TIMED_WAIT = config['SEM_TIMED_WAIT']
     SEM_T_SIZE = config['SEM_T_SIZE']
     if sys.platform == 'darwin':
diff --git a/rpython/jit/backend/detect_cpu.py b/rpython/jit/backend/detect_cpu.py
--- a/rpython/jit/backend/detect_cpu.py
+++ b/rpython/jit/backend/detect_cpu.py
@@ -57,6 +57,7 @@
             'i486': MODEL_X86,
             'i586': MODEL_X86,
             'i686': MODEL_X86,
+            'i686-AT386': MODEL_X86,  # Hurd
             'i86pc': MODEL_X86,    # Solaris/Intel
             'x86': MODEL_X86,      # Apple
             'Power Macintosh': MODEL_PPC_64,
diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py
--- a/rpython/rlib/rposix.py
+++ b/rpython/rlib/rposix.py
@@ -1086,9 +1086,12 @@
         else:
             return bool(c_func(status))
 
-WAIT_MACROS = ['WCOREDUMP', 'WIFCONTINUED', 'WIFSTOPPED',
+WAIT_MACROS = ['WCOREDUMP', 'WIFSTOPPED',
                'WIFSIGNALED', 'WIFEXITED',
                'WEXITSTATUS', 'WSTOPSIG', 'WTERMSIG']
+if not sys.platform.startswith('gnu'):
+    WAIT_MACROS.append('WIFCONTINUED')
+
 for name in WAIT_MACROS:
     _make_waitmacro(name)
 
diff --git a/rpython/translator/platform/__init__.py b/rpython/translator/platform/__init__.py
--- a/rpython/translator/platform/__init__.py
+++ b/rpython/translator/platform/__init__.py
@@ -308,6 +308,13 @@
         host_factory = OpenBSD
     else:
         host_factory = OpenBSD_64
+elif sys.platform.startswith('gnu'):
+    from rpython.translator.platform.hurd import Hurd
+    import platform
+    if platform.architecture()[0] == '32bit':
+        host_factory = Hurd
+    else:
+        host_factory = Hurd_64
 elif os.name == 'nt':
     from rpython.translator.platform.windows import Windows, Windows_x64
     import platform
diff --git a/rpython/translator/platform/hurd.py b/rpython/translator/platform/hurd.py
new file mode 100644
--- /dev/null
+++ b/rpython/translator/platform/hurd.py
@@ -0,0 +1,42 @@
+"""Support for Hurd."""
+
+import os
+import platform
+import sys
+from rpython.translator.platform.posix import BasePosix
+
+class BaseHurd(BasePosix):
+    name = "hurd"
+
+    link_flags = tuple(
+                 ['-pthread',]
+                 + os.environ.get('LDFLAGS', '').split())
+    extra_libs = ('-lrt',)
+    cflags = tuple(
+             ['-O3', '-pthread', '-fomit-frame-pointer',
+              '-Wall', '-Wno-unused', '-Wno-address']
+             + os.environ.get('CFLAGS', '').split())
+    standalone_only = ()
+    shared_only = ('-fPIC',)
+    so_ext = 'so'
+
+    def _args_for_shared(self, args, **kwds):
+        return ['-shared'] + args
+
+    def _include_dirs_for_libffi(self):
+        return self._pkg_config("libffi", "--cflags-only-I",
+                                ['/usr/include/libffi'],
+                                check_result_dir=True)
+
+    def _library_dirs_for_libffi(self):
+        return self._pkg_config("libffi", "--libs-only-L",
+                                ['/usr/lib/libffi'],
+                                check_result_dir=True)
+
+
+class Hurd(BaseHurd):
+    shared_only = () # it seems that on 32-bit GNU, compiling with -fPIC
+                     # gives assembler that asmgcc is not happy about.
+
+class HurdPIC(BaseHurd):
+    pass

From pypy.commits at gmail.com  Mon Jul 23 11:34:57 2018
From: pypy.commits at gmail.com (arigo)
Date: Mon, 23 Jul 2018 08:34:57 -0700 (PDT)
Subject: [pypy-commit] pypy default: Issue #2853
Message-ID: <5b55f5a1.1c69fb81.afc0f.e62f@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r94881:24a43b6a4d73
Date: 2018-07-23 17:33 +0200
http://bitbucket.org/pypy/pypy/changeset/24a43b6a4d73/

Log:	Issue #2853

	Patch by Babak F.

diff --git a/rpython/rlib/rvmprof/cintf.py b/rpython/rlib/rvmprof/cintf.py
--- a/rpython/rlib/rvmprof/cintf.py
+++ b/rpython/rlib/rvmprof/cintf.py
@@ -17,7 +17,7 @@
 
 # vmprof works only on x86 for now
 IS_SUPPORTED = False
-if sys.platform in ('darwin', 'linux', 'linux2', 'freebsd10', 'freebsd11'):
+if sys.platform in ('darwin', 'linux', 'linux2') or sys.platform.startswith('freebsd'):
     try:
         IS_SUPPORTED = detect_cpu.autodetect().startswith('x86')
     except detect_cpu.ProcessorAutodetectError:

From pypy.commits at gmail.com  Mon Jul 23 12:11:38 2018
From: pypy.commits at gmail.com (arigo)
Date: Mon, 23 Jul 2018 09:11:38 -0700 (PDT)
Subject: [pypy-commit] pypy default: Give a warning when we can't import
 _hashlib
Message-ID: <5b55fe3a.1c69fb81.3a47d.784a@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r94882:f59578325003
Date: 2018-07-23 18:10 +0200
http://bitbucket.org/pypy/pypy/changeset/f59578325003/

Log:	Give a warning when we can't import _hashlib

diff --git a/lib-python/2.7/hashlib.py b/lib-python/2.7/hashlib.py
--- a/lib-python/2.7/hashlib.py
+++ b/lib-python/2.7/hashlib.py
@@ -136,9 +136,14 @@
     __get_hash = __get_openssl_constructor
     algorithms_available = algorithms_available.union(
         _hashlib.openssl_md_meth_names)
-except ImportError:
+except ImportError as e:
     new = __py_new
     __get_hash = __get_builtin_constructor
+    # added by PyPy
+    import warnings
+    warnings.warn("The _hashlib module is not available, falling back "
+                  "to a much slower implementation (%s)" % str(e),
+                  RuntimeWarning)
 
 for __func_name in __always_supported:
     # try them all, some may not work due to the OpenSSL

From pypy.commits at gmail.com  Mon Jul 23 12:11:40 2018
From: pypy.commits at gmail.com (arigo)
Date: Mon, 23 Jul 2018 09:11:40 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Give a warning when we can't import
 _hashlib
Message-ID: <5b55fe3c.1c69fb81.52565.9f4a@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r94883:f21562687ea4
Date: 2018-07-23 18:10 +0200
http://bitbucket.org/pypy/pypy/changeset/f21562687ea4/

Log:	Give a warning when we can't import _hashlib

diff --git a/lib-python/3/hashlib.py b/lib-python/3/hashlib.py
--- a/lib-python/3/hashlib.py
+++ b/lib-python/3/hashlib.py
@@ -134,9 +134,14 @@
     __get_hash = __get_openssl_constructor
     algorithms_available = algorithms_available.union(
             _hashlib.openssl_md_meth_names)
-except ImportError:
+except ImportError as e:
     new = __py_new
     __get_hash = __get_builtin_constructor
+    # added by PyPy
+    import warnings
+    warnings.warn("The _hashlib module is not available, falling back "
+                  "to a much slower implementation (%s)" % str(e),
+                  RuntimeWarning)
 
 try:
     # OpenSSL's PKCS5_PBKDF2_HMAC requires OpenSSL 1.0+ with HMAC and SHA

From pypy.commits at gmail.com  Mon Jul 23 12:19:33 2018
From: pypy.commits at gmail.com (arigo)
Date: Mon, 23 Jul 2018 09:19:33 -0700 (PDT)
Subject: [pypy-commit] pypy default: Add a get_raw_address() in the base
 class---better than segfaulting due
Message-ID: <5b560015.1c69fb81.12f99.2511@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r94884:6868da604e68
Date: 2018-07-23 18:18 +0200
http://bitbucket.org/pypy/pypy/changeset/6868da604e68/

Log:	Add a get_raw_address() in the base class---better than segfaulting
	due to calls to a missing method!

diff --git a/rpython/rlib/buffer.py b/rpython/rlib/buffer.py
--- a/rpython/rlib/buffer.py
+++ b/rpython/rlib/buffer.py
@@ -6,7 +6,7 @@
 from rpython.rtyper.lltypesystem.rstr import STR
 from rpython.rtyper.lltypesystem.rlist import LIST_OF
 from rpython.rtyper.annlowlevel import llstr
-from rpython.rlib.objectmodel import specialize
+from rpython.rlib.objectmodel import specialize, we_are_translated
 from rpython.rlib import jit
 from rpython.rlib.rgc import (resizable_list_supporting_raw_ptr,
                               nonmoving_raw_ptr_for_resizable_list,
@@ -114,6 +114,12 @@
         """
         raise CannotWrite
 
+    def get_raw_address(self):
+        msg = "cannot take the raw address of this buffer"
+        if not we_are_translated():
+            msg += " '%s'" % (self,)
+        raise ValueError(msg)
+
 
 class RawBuffer(Buffer):
     """

From pypy.commits at gmail.com  Mon Jul 23 12:31:05 2018
From: pypy.commits at gmail.com (arigo)
Date: Mon, 23 Jul 2018 09:31:05 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: hg merge default
Message-ID: <5b5602c9.1c69fb81.83af1.5dec@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r94885:130fbb74dab5
Date: 2018-07-23 18:20 +0200
http://bitbucket.org/pypy/pypy/changeset/130fbb74dab5/

Log:	hg merge default

diff too long, truncating to 2000 out of 2216 lines

diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py
--- a/pypy/config/pypyoption.py
+++ b/pypy/config/pypyoption.py
@@ -43,14 +43,10 @@
     "_jitlog",
 ])
 
-from rpython.jit.backend import detect_cpu
-try:
-    if detect_cpu.autodetect().startswith('x86'):
-        if not sys.platform.startswith('openbsd'):
-            working_modules.add('_vmprof')
-            working_modules.add('faulthandler')
-except detect_cpu.ProcessorAutodetectError:
-    pass
+import rpython.rlib.rvmprof.cintf
+if rpython.rlib.rvmprof.cintf.IS_SUPPORTED:
+    working_modules.add('_vmprof')
+    working_modules.add('faulthandler')
 
 translation_modules = default_modules.copy()
 translation_modules.update([
@@ -323,3 +319,4 @@
     parser = to_optparse(config) #, useoptions=["translation.*"])
     option, args = parser.parse_args()
     print config
+    print working_modules
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -7,9 +7,12 @@
 
 .. branch: cppyy-packaging
 
-Upgrade to backend 1.1.0, improved handling of templated methods and
+Upgrade to backend 1.2.0, improved handling of templated methods and
 functions (in particular automatic deduction of types), improved pythonization
-interface, and a range of compatibility fixes for Python3
+interface, range of compatibility fixes for Python3, free functions now take
+fast libffi path when possible, moves for strings (incl. from Python str),
+easier/faster handling of std::vector by numpy, improved and faster object
+identity preservation
 
 .. branch: socket_default_timeout_blockingness
 
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -3935,8 +3935,8 @@
 
 def test_char_pointer_conversion():
     import warnings
-    assert __version__.startswith(("1.8", "1.9", "1.10", "1.11", "1.12")), (
-        "consider turning the warning into an error")
+    assert __version__.startswith("1."), (
+        "the warning will be an error if we ever release cffi 2.x")
     BCharP = new_pointer_type(new_primitive_type("char"))
     BIntP = new_pointer_type(new_primitive_type("int"))
     BVoidP = new_pointer_type(new_void_type())
diff --git a/pypy/module/_cppyy/converter.py b/pypy/module/_cppyy/converter.py
--- a/pypy/module/_cppyy/converter.py
+++ b/pypy/module/_cppyy/converter.py
@@ -1,15 +1,12 @@
 import sys
 
 from pypy.interpreter.error import OperationError, oefmt
-
 from rpython.rtyper.lltypesystem import rffi, lltype
 from rpython.rlib.rarithmetic import r_singlefloat, r_longfloat
 from rpython.rlib import rfloat, rawrefcount
-
 from pypy.module._rawffi.interp_rawffi import letter2tp
 from pypy.module._rawffi.array import W_ArrayInstance
-
-from pypy.module._cppyy import helper, capi, ffitypes
+from pypy.module._cppyy import helper, capi, ffitypes, lowlevelviews
 
 # Converter objects are used to translate between RPython and C++. They are
 # defined by the type name for which they provide conversion. Uses are for
@@ -149,7 +146,8 @@
         # read access, so no copy needed
         address_value = self._get_raw_address(space, w_obj, offset)
         address = rffi.cast(rffi.ULONG, address_value)
-        return W_ArrayInstance(space, letter2tp(space, self.typecode), self.size, address)
+        return lowlevelviews.W_LowLevelView(
+            space, letter2tp(space, self.typecode), self.size, address)
 
     def to_memory(self, space, w_obj, w_value, offset):
         # copy the full array (uses byte copy for now)
@@ -190,7 +188,8 @@
         # read access, so no copy needed
         address_value = self._get_raw_address(space, w_obj, offset)
         address = rffi.cast(rffi.ULONGP, address_value)
-        return W_ArrayInstance(space, letter2tp(space, self.typecode), self.size, address[0])
+        return lowlevelviews.W_LowLevelView(
+            space, letter2tp(space, self.typecode), self.size, address[0])
 
     def to_memory(self, space, w_obj, w_value, offset):
         # copy only the pointer value
@@ -438,7 +437,7 @@
             from pypy.module._cppyy import interp_cppyy
             return interp_cppyy.get_nullptr(space)
         shape = letter2tp(space, 'P')
-        return W_ArrayInstance(space, shape, sys.maxint/shape.size, ptrval)
+        return lowlevelviews.W_LowLevelView(space, shape, sys.maxint/shape.size, ptrval)
 
     def to_memory(self, space, w_obj, w_value, offset):
         address = rffi.cast(rffi.VOIDPP, self._get_raw_address(space, w_obj, offset))
@@ -491,7 +490,7 @@
         from pypy.module._cppyy.interp_cppyy import W_CPPInstance
         if isinstance(w_obj, W_CPPInstance):
             from pypy.module._cppyy.interp_cppyy import INSTANCE_FLAGS_IS_RVALUE
-            if w_obj.flags & INSTANCE_FLAGS_IS_RVALUE:
+            if w_obj.rt_flags & INSTANCE_FLAGS_IS_RVALUE:
                 # reject moves as all are explicit
                 raise ValueError("lvalue expected")
             if capi.c_is_subtype(space, w_obj.clsdecl, self.clsdecl):
@@ -523,14 +522,14 @@
         from pypy.module._cppyy.interp_cppyy import W_CPPInstance, INSTANCE_FLAGS_IS_RVALUE
         obj = space.interp_w(W_CPPInstance, w_obj)
         if obj:
-            if obj.flags & INSTANCE_FLAGS_IS_RVALUE:
-                obj.flags &= ~INSTANCE_FLAGS_IS_RVALUE
+            if obj.rt_flags & INSTANCE_FLAGS_IS_RVALUE:
+                obj.rt_flags &= ~INSTANCE_FLAGS_IS_RVALUE
                 try:
                     return InstanceRefConverter._unwrap_object(self, space, w_obj)
                 except Exception:
                     # TODO: if the method fails on some other converter, then the next
                     # overload can not be an rvalue anymore
-                    obj.flags |= INSTANCE_FLAGS_IS_RVALUE
+                    obj.rt_flags |= INSTANCE_FLAGS_IS_RVALUE
                     raise
         raise oefmt(space.w_ValueError, "object is not an rvalue")
 
@@ -566,10 +565,6 @@
         from pypy.module._cppyy import interp_cppyy
         return interp_cppyy.wrap_cppinstance(space, address, self.clsdecl, do_cast=False)
 
-    def to_memory(self, space, w_obj, w_value, offset):
-        address = rffi.cast(rffi.VOIDPP, self._get_raw_address(space, w_obj, offset))
-        address[0] = rffi.cast(rffi.VOIDP, self._unwrap_object(space, w_value))
-
 class InstancePtrPtrConverter(InstancePtrConverter):
     typecode = 'o'
 
@@ -597,6 +592,25 @@
         return interp_cppyy.wrap_cppinstance(
             space, address, self.clsdecl, do_cast=False, is_ref=True)
 
+    def to_memory(self, space, w_obj, w_value, offset):
+        # the actual data member is of object* type, but we receive a pointer to that
+        # data member in order to modify its value, so by convention, the internal type
+        # used is object**
+        address = rffi.cast(rffi.VOIDPP, self._get_raw_address(space, w_obj, offset))
+        from pypy.module._cppyy.interp_cppyy import W_CPPInstance
+        cppinstance = space.interp_w(W_CPPInstance, w_value, can_be_None=True)
+        if cppinstance:
+            rawobject = cppinstance.get_rawobject()
+            offset = capi.c_base_offset(space, cppinstance.clsdecl, self.clsdecl, rawobject, 1)
+            obj_address = capi.direct_ptradd(rawobject, offset)
+            address[0] = rffi.cast(rffi.VOIDP, obj_address);
+            # register the value for potential recycling
+            from pypy.module._cppyy.interp_cppyy import memory_regulator
+            memory_regulator.register(cppinstance)
+        else:
+            raise oefmt(space.w_TypeError,
+                "cannot pass %T instance as %s", w_value, self.clsdecl.name)
+
     def finalize_call(self, space, w_obj):
         if self.ref_buffer:
             set_rawobject(space, w_obj, self.ref_buffer[0])
@@ -607,7 +621,6 @@
             self.ref_buffer = lltype.nullptr(rffi.VOIDPP.TO)
 
 class StdStringConverter(InstanceConverter):
-
     def __init__(self, space, extra):
         from pypy.module._cppyy import interp_cppyy
         cppclass = interp_cppyy.scope_byname(space, capi.std_string_name)
@@ -633,6 +646,34 @@
     def free_argument(self, space, arg):
         capi.c_destruct(space, self.clsdecl, rffi.cast(capi.C_OBJECT, rffi.cast(rffi.VOIDPP, arg)[0]))
 
+class StdStringMoveConverter(StdStringConverter):
+    def _unwrap_object(self, space, w_obj):
+        # moving is same as by-ref, but have to check that move is allowed
+        moveit_reason = 3
+        from pypy.module._cppyy.interp_cppyy import W_CPPInstance, INSTANCE_FLAGS_IS_RVALUE
+        try:
+            obj = space.interp_w(W_CPPInstance, w_obj)
+            if obj and obj.rt_flags & INSTANCE_FLAGS_IS_RVALUE:
+                obj.rt_flags &= ~INSTANCE_FLAGS_IS_RVALUE
+                moveit_reason = 1
+            else:
+                moveit_reason = 0
+        except:
+            pass
+
+        if moveit_reason:
+            try:
+                return StdStringConverter._unwrap_object(self, space, w_obj)
+            except Exception:
+                 if moveit_reason == 1:
+                    # TODO: if the method fails on some other converter, then the next
+                    # overload can not be an rvalue anymore
+                    obj = space.interp_w(W_CPPInstance, w_obj)
+                    obj.rt_flags |= INSTANCE_FLAGS_IS_RVALUE
+                    raise
+
+        raise oefmt(space.w_ValueError, "object is not an rvalue")
+
 class StdStringRefConverter(InstancePtrConverter):
     _immutable_fields_ = ['cppclass', 'typecode']
     typecode    = 'V'
@@ -885,6 +926,7 @@
 _converters["std::basic_string<char>"]           = StdStringConverter
 _converters["const std::basic_string<char>&"]    = StdStringConverter     # TODO: shouldn't copy
 _converters["std::basic_string<char>&"]          = StdStringRefConverter
+_converters["std::basic_string<char>&&"]         = StdStringMoveConverter
 
 _converters["PyObject*"]                         = PyObjectConverter
 
@@ -1002,6 +1044,7 @@
         ("std::basic_string<char>",         "string"),
         ("const std::basic_string<char>&",  "const string&"),
         ("std::basic_string<char>&",        "string&"),
+        ("std::basic_string<char>&&",       "string&&"),
 
         ("PyObject*",                       "_object*"),
     )
diff --git a/pypy/module/_cppyy/executor.py b/pypy/module/_cppyy/executor.py
--- a/pypy/module/_cppyy/executor.py
+++ b/pypy/module/_cppyy/executor.py
@@ -1,14 +1,10 @@
 import sys
 
 from pypy.interpreter.error import oefmt
-
 from rpython.rtyper.lltypesystem import rffi, lltype
 from rpython.rlib import jit_libffi
-
 from pypy.module._rawffi.interp_rawffi import letter2tp
-from pypy.module._rawffi.array import W_Array, W_ArrayInstance
-
-from pypy.module._cppyy import helper, capi, ffitypes
+from pypy.module._cppyy import helper, capi, ffitypes, lowlevelviews
 
 # Executor objects are used to dispatch C++ methods. They are defined by their
 # return type only: arguments are converted by Converter objects, and Executors
@@ -60,7 +56,7 @@
             from pypy.module._cppyy import interp_cppyy
             return interp_cppyy.get_nullptr(space)
         shape = letter2tp(space, self.typecode)
-        return W_ArrayInstance(space, shape, sys.maxint/shape.size, ptrval)
+        return lowlevelviews.W_LowLevelView(space, shape, sys.maxint/shape.size, ptrval)
 
 
 class VoidExecutor(Executor):
@@ -98,10 +94,10 @@
     def __init__(self, space, extra):
         Executor.__init__(self, space, extra)
         self.do_assign = False
-        self.item = rffi.cast(self.c_type, 0)
+        self.w_item = space.w_None
 
     def set_item(self, space, w_item):
-        self.item = self._unwrap_object(space, w_item)
+        self.w_item = w_item
         self.do_assign = True
 
     #def _wrap_object(self, space, obj):
@@ -109,7 +105,7 @@
 
     def _wrap_reference(self, space, rffiptr):
         if self.do_assign:
-            rffiptr[0] = self.item
+            rffiptr[0] = rffi.cast(self.c_type, self._unwrap_object(space, self.w_item))
         self.do_assign = False
         return self._wrap_object(space, rffiptr[0])    # all paths, for rtyper
 
diff --git a/pypy/module/_cppyy/interp_cppyy.py b/pypy/module/_cppyy/interp_cppyy.py
--- a/pypy/module/_cppyy/interp_cppyy.py
+++ b/pypy/module/_cppyy/interp_cppyy.py
@@ -41,6 +41,7 @@
              'void**'        : 100,
              'float'         :  30,
              'double'        :  10,
+             'bool'          :   1,
              'const string&' :   1, } # solves a specific string ctor overload
 
 from rpython.rlib.listsort import make_timsort_class
@@ -167,6 +168,7 @@
 #
 #  W_CPPOverload:                 instance methods (base class)
 #  W_CPPConstructorOverload:      constructors
+#  W_CPPAbstractCtorOverload:     to provent instantiation of abstract classes
 #  W_CPPStaticOverload:           free and static functions
 #  W_CPPTemplateOverload:         templated methods
 #  W_CPPTemplateStaticOverload:   templated free and static functions
@@ -258,7 +260,8 @@
         jit.promote(self)
         cif_descr = self.cif_descr
         # add extra space for const-ref support (see converter.py)
-        buffer = lltype.malloc(rffi.CCHARP.TO, cif_descr.exchange_size+len(self.arg_defs)*rffi.sizeof(rffi.DOUBLE), flavor='raw')
+        buffer = lltype.malloc(rffi.CCHARP.TO,
+            cif_descr.exchange_size+len(self.arg_defs)*rffi.sizeof(rffi.DOUBLE), flavor='raw')
         thisoff = 0
         try:
             if cppthis:
@@ -412,8 +415,10 @@
 
     def priority(self):
         total_arg_priority = 0
-        for p in [priority.get(arg_type, 0) for arg_type, arg_dflt in self.arg_defs]:
-            total_arg_priority += p
+        for arg_type, arg_dflt in self.arg_defs:
+            total_arg_priority += priority.get(arg_type, 0)
+            if '&&' in arg_type:
+                total_arg_priority += 100
         return total_arg_priority
 
     @rgc.must_be_light_finalizer
@@ -433,7 +438,7 @@
 
 class CPPSetItem(CPPMethod):
     """Method dispatcher specific to Python's __setitem__ mapped onto C++'s
-    operator[](int). The former function takes an extra argument to assign to
+    operator[](T). The former function takes an extra argument to assign to
     the return type of the latter."""
 
     _attrs_ = []
@@ -502,6 +507,9 @@
     def descr_get(self, w_obj, w_cls=None):
         """functionobject.__get__(obj[, type]) -> method"""
         # TODO: check validity of w_cls if given
+        # TODO: this does not work for Python 3, which does not have
+        #  unbound methods (probably no common code possible, see also
+        #  pypy/interpreter/function.py)
         space = self.space
         asking_for_bound = (space.is_none(w_cls) or
                             not space.is_w(w_obj, space.w_None) or
@@ -581,6 +589,16 @@
             sig += '\n'+self.functions[i].prototype()
         return self.space.newtext(sig)
 
+    @unwrap_spec(signature='text')
+    def mp_overload(self, signature):
+        sig = '(%s)' % signature
+        for f in self.functions:
+            if f.signature(False) == sig:
+                if isinstance(self, W_CPPStaticOverload):
+                    return W_CPPStaticOverload(self.space, self.scope, [f])
+                return W_CPPOverload(self.space, self.scope, [f])
+        raise oefmt(self.space.w_LookupError, "signature '%s' not found", signature)
+
     # allow user to determine ffi use rules per overload
     def fget_useffi(self, space):
         return space.newbool(bool(self.flags & OVERLOAD_FLAGS_USE_FFI))
@@ -604,10 +622,11 @@
 
 W_CPPOverload.typedef = TypeDef(
     'CPPOverload',
-    __get__    = interp2app(W_CPPOverload.descr_get),
-    __call__   = interp2app(W_CPPOverload.call_args),
-    __useffi__ = GetSetProperty(W_CPPOverload.fget_useffi, W_CPPOverload.fset_useffi),
-    __doc__    = GetSetProperty(W_CPPOverload.fget_doc)
+    __get__      = interp2app(W_CPPOverload.descr_get),
+    __call__     = interp2app(W_CPPOverload.call_args),
+    __useffi__   = GetSetProperty(W_CPPOverload.fget_useffi, W_CPPOverload.fset_useffi),
+    __overload__ = interp2app(W_CPPOverload.mp_overload),
+    __doc__      = GetSetProperty(W_CPPOverload.fget_doc)
 )
 
 
@@ -629,21 +648,18 @@
     @unwrap_spec(args_w='args_w')
     def call_args(self, args_w):
         jit.promote(self)
-        #if isinstance(args_w[0], W_CPPInstance):
-            # free function used as bound method, leave in place
         return self.call_impl(capi.C_NULL_OBJECT, args_w)
-        # free functions are implemented as methods of 'namespace' classes, remove 'instance'
-        #return self.call_impl(capi.C_NULL_OBJECT, args_w[1:])
 
     def __repr__(self):
         return "W_CPPStaticOverload(%s)" % [f.prototype() for f in self.functions]
 
 W_CPPStaticOverload.typedef = TypeDef(
     'CPPStaticOverload',
-    __get__    = interp2app(W_CPPStaticOverload.descr_get),
-    __call__   = interp2app(W_CPPStaticOverload.call_args),
-    __useffi__ = GetSetProperty(W_CPPStaticOverload.fget_useffi, W_CPPStaticOverload.fset_useffi),
-    __doc__    = GetSetProperty(W_CPPStaticOverload.fget_doc)
+    __get__      = interp2app(W_CPPStaticOverload.descr_get),
+    __call__     = interp2app(W_CPPStaticOverload.call_args),
+    __useffi__   = GetSetProperty(W_CPPStaticOverload.fget_useffi, W_CPPStaticOverload.fset_useffi),
+    __overload__ = interp2app(W_CPPStaticOverload.mp_overload),
+    __doc__      = GetSetProperty(W_CPPStaticOverload.fget_doc)
 )
 
 
@@ -657,11 +673,6 @@
     @unwrap_spec(args_w='args_w')
     def call_args(self, args_w):
         jit.promote(self)
-        # TODO: factor out the following:
-        if capi.c_is_abstract(self.space, self.scope.handle):
-            raise oefmt(self.space.w_TypeError,
-                        "cannot instantiate abstract class '%s'",
-                        self.scope.name)
         cppinstance = self.space.interp_w(W_CPPInstance, args_w[0])
         w_result = self.call_impl(rffi.cast(capi.C_OBJECT, self.scope.handle), args_w[1:])
         newthis = rffi.cast(capi.C_OBJECT, self.space.uint_w(w_result))
@@ -674,9 +685,27 @@
 
 W_CPPConstructorOverload.typedef = TypeDef(
     'CPPConstructorOverload',
-    __get__    = interp2app(W_CPPConstructorOverload.descr_get),
-    __call__   = interp2app(W_CPPConstructorOverload.call_args),
-    __doc__    = GetSetProperty(W_CPPConstructorOverload.fget_doc)
+    __get__      = interp2app(W_CPPConstructorOverload.descr_get),
+    __call__     = interp2app(W_CPPConstructorOverload.call_args),
+    __overload__ = interp2app(W_CPPConstructorOverload.mp_overload),
+    __doc__      = GetSetProperty(W_CPPConstructorOverload.fget_doc)
+)
+
+class W_CPPAbstractCtorOverload(W_CPPOverload):
+    _attrs_ = []
+
+    @unwrap_spec(args_w='args_w')
+    def call_args(self, args_w):
+        raise oefmt(self.space.w_TypeError,
+            "cannot instantiate abstract class '%s'", self.scope.name)
+
+    def __repr__(self):
+        return "W_CPPAbstractCtorOverload"
+
+W_CPPAbstractCtorOverload.typedef = TypeDef(
+    'CPPAbstractCtorOverload',
+    __get__    = interp2app(W_CPPAbstractCtorOverload.descr_get),
+    __call__   = interp2app(W_CPPAbstractCtorOverload.call_args),
 )
 
 
@@ -699,7 +728,7 @@
                     if args_w:
                         # try to specialize the type match for the given object
                         cppinstance = self.space.interp_w(W_CPPInstance, args_w[i])
-                        if cppinstance.flags & INSTANCE_FLAGS_IS_RVALUE:
+                        if cppinstance.rt_flags & INSTANCE_FLAGS_IS_RVALUE:
                             sugar = "&&"
                         elif cppinstance.flags & INSTANCE_FLAGS_IS_REF:
                             sugar = "*"
@@ -839,11 +868,11 @@
 
 W_CPPTemplateOverload.typedef = TypeDef(
     'CPPTemplateOverload',
-    __get__     = interp2app(W_CPPTemplateOverload.descr_get),
-    __getitem__ = interp2app(W_CPPTemplateOverload.getitem),
-    __call__    = interp2app(W_CPPTemplateOverload.call_args),
-    __useffi__  = GetSetProperty(W_CPPTemplateOverload.fget_useffi, W_CPPTemplateOverload.fset_useffi),
-    __doc__     = GetSetProperty(W_CPPTemplateOverload.fget_doc)
+    __get__      = interp2app(W_CPPTemplateOverload.descr_get),
+    __getitem__  = interp2app(W_CPPTemplateOverload.getitem),
+    __call__     = interp2app(W_CPPTemplateOverload.call_args),
+    __useffi__   = GetSetProperty(W_CPPTemplateOverload.fget_useffi, W_CPPTemplateOverload.fset_useffi),
+    __doc__      = GetSetProperty(W_CPPTemplateOverload.fget_doc)
 )
 
 class W_CPPTemplateStaticOverload(W_CPPStaticOverload, TemplateOverloadMixin):
@@ -897,11 +926,11 @@
 
 W_CPPTemplateStaticOverload.typedef = TypeDef(
     'CPPTemplateStaticOverload',
-    __get__     = interp2app(W_CPPTemplateStaticOverload.descr_get),
-    __getitem__ = interp2app(W_CPPTemplateStaticOverload.getitem),
-    __call__    = interp2app(W_CPPTemplateStaticOverload.call_args),
-    __useffi__  = GetSetProperty(W_CPPTemplateStaticOverload.fget_useffi, W_CPPTemplateStaticOverload.fset_useffi),
-    __doc__     = GetSetProperty(W_CPPTemplateStaticOverload.fget_doc)
+    __get__      = interp2app(W_CPPTemplateStaticOverload.descr_get),
+    __getitem__  = interp2app(W_CPPTemplateStaticOverload.getitem),
+    __call__     = interp2app(W_CPPTemplateStaticOverload.call_args),
+    __useffi__   = GetSetProperty(W_CPPTemplateStaticOverload.fget_useffi, W_CPPTemplateStaticOverload.fset_useffi),
+    __doc__      = GetSetProperty(W_CPPTemplateStaticOverload.fget_doc)
 )
 
 
@@ -929,6 +958,7 @@
 
     def _get_offset(self, cppinstance):
         if cppinstance:
+            assert isinstance(cppinstance.clsdecl, W_CPPClassDecl)
             assert lltype.typeOf(cppinstance.clsdecl.handle) == lltype.typeOf(self.scope.handle)
             offset = self.offset + cppinstance.clsdecl.get_base_offset(cppinstance, self.scope)
         else:
@@ -1074,6 +1104,8 @@
         sig = '(%s)' % signature
         for f in overload.functions:
             if f.signature(False) == sig:
+                if isinstance(overload, W_CPPStaticOverload):
+                    return W_CPPStaticOverload(self.space, self, [f])
                 return W_CPPOverload(self.space, self, [f])
         raise oefmt(self.space.w_LookupError, "no overload matches signature")
 
@@ -1169,9 +1201,13 @@
 
 
 class W_CPPClassDecl(W_CPPScopeDecl):
-    _attrs_ = ['space', 'handle', 'name', 'overloads', 'datamembers']
+    _attrs_ = ['space', 'handle', 'name', 'overloads', 'datamembers', 'cppobjects']
     _immutable_fields_ = ['handle', 'name', 'overloads[*]', 'datamembers[*]']
 
+    def __init__(self, space, opaque_handle, final_scoped_name):
+        W_CPPScopeDecl.__init__(self, space, opaque_handle, final_scoped_name)
+        self.cppobjects = rweakref.RWeakValueDictionary(int, W_CPPInstance)
+
     def _build_overloads(self):
         assert len(self.overloads) == 0
         methods_tmp = {}; ftype_tmp = {}
@@ -1210,7 +1246,10 @@
             ftype = ftype_tmp[pyname]
             CPPMethodSort(methods).sort()
             if ftype & FUNCTION_IS_CONSTRUCTOR:
-                overload = W_CPPConstructorOverload(self.space, self, methods[:])
+                if capi.c_is_abstract(self.space, self.handle):
+                    overload = W_CPPAbstractCtorOverload(self.space, self, methods[:])
+                else:
+                    overload = W_CPPConstructorOverload(self.space, self, methods[:])
             elif ftype & FUNCTION_IS_STATIC:
                 if ftype & FUNCTION_IS_TEMPLATE:
                     cppname = capi.c_method_name(self.space, methods[0].cppmethod)
@@ -1276,10 +1315,12 @@
         raise self.missing_attribute_error(name)
 
     def get_base_offset(self, cppinstance, calling_scope):
+        assert isinstance(cppinstance.clsdecl, W_CPPClassDecl)
         assert self == cppinstance.clsdecl
         return 0
 
     def get_cppthis(self, cppinstance, calling_scope):
+        assert isinstance(cppinstance.clsdecl, W_CPPClassDecl)
         assert self == cppinstance.clsdecl
         return cppinstance.get_rawobject()
 
@@ -1315,12 +1356,14 @@
 
 class W_CPPComplexClassDecl(W_CPPClassDecl):
     def get_base_offset(self, cppinstance, calling_scope):
+        assert isinstance(cppinstance.clsdecl, W_CPPComplexClassDecl)
         assert self == cppinstance.clsdecl
         offset = capi.c_base_offset(self.space,
-                                    self, calling_scope, cppinstance.get_rawobject(), 1)
+            self, calling_scope, cppinstance.get_rawobject(), 1)
         return offset
 
     def get_cppthis(self, cppinstance, calling_scope):
+        assert isinstance(cppinstance.clsdecl, W_CPPComplexClassDecl)
         assert self == cppinstance.clsdecl
         offset = self.get_base_offset(cppinstance, calling_scope)
         return capi.direct_ptradd(cppinstance.get_rawobject(), offset)
@@ -1340,9 +1383,9 @@
 
 
 class W_CPPInstance(W_Root):
-    _attrs_ = ['space', 'clsdecl', '_rawobject', 'smartdecl', 'deref', 'flags',
+    _attrs_ = ['space', 'clsdecl', '_rawobject', 'smartdecl', 'deref', 'flags', 'rt_flags',
                'finalizer_registered']
-    _immutable_fields_ = ['clsdecl', 'smartdecl', 'deref']
+    _immutable_fields_ = ['clsdecl', 'smartdecl', 'deref', 'flags']
 
     finalizer_registered = False
 
@@ -1350,6 +1393,7 @@
                  smartdecl=None, deref=rffi.cast(capi.C_METHOD, 0)):
         self.space = space
         self.clsdecl = decl
+        assert isinstance(self.clsdecl, W_CPPClassDecl)
         assert lltype.typeOf(rawobject) == capi.C_OBJECT
         assert not isref or rawobject
         self._rawobject = rawobject
@@ -1357,15 +1401,16 @@
         self.flags = 0
         if isref or (smartdecl and deref):
             self.flags |= INSTANCE_FLAGS_IS_REF
+        self.rt_flags = 0
         if python_owns:
-            self.flags |= INSTANCE_FLAGS_PYTHON_OWNS
+            self.rt_flags |= INSTANCE_FLAGS_PYTHON_OWNS
             self._opt_register_finalizer()
         self.smartdecl = smartdecl
         self.deref     = deref
 
     def _opt_register_finalizer(self):
         if not self.finalizer_registered and not hasattr(self.space, "fake"):
-            assert self.flags & INSTANCE_FLAGS_PYTHON_OWNS
+            assert self.rt_flags & INSTANCE_FLAGS_PYTHON_OWNS
             self.register_finalizer(self.space)
             self.finalizer_registered = True
 
@@ -1377,17 +1422,18 @@
 
     # allow user to determine ownership rules on a per object level
     def fget_python_owns(self, space):
-        return space.newbool(bool(self.flags & INSTANCE_FLAGS_PYTHON_OWNS))
+        return space.newbool(bool(self.rt_flags & INSTANCE_FLAGS_PYTHON_OWNS))
 
     @unwrap_spec(value=bool)
     def fset_python_owns(self, space, value):
         if space.is_true(value):
-            self.flags |= INSTANCE_FLAGS_PYTHON_OWNS
+            self.rt_flags |= INSTANCE_FLAGS_PYTHON_OWNS
             self._opt_register_finalizer()
         else:
-            self.flags &= ~INSTANCE_FLAGS_PYTHON_OWNS
+            self.rt_flags &= ~INSTANCE_FLAGS_PYTHON_OWNS
 
     def get_cppthis(self, calling_scope):
+        assert isinstance(self.clsdecl, W_CPPClassDecl)
         return self.clsdecl.get_cppthis(self, calling_scope)
 
     def get_rawobject(self):
@@ -1494,6 +1540,7 @@
 
     def destruct(self):
         if self._rawobject:
+            assert isinstance(self.clsdecl, W_CPPClassDecl)
             if self.smartdecl and self.deref:
                 klass = self.smartdecl
             elif not (self.flags & INSTANCE_FLAGS_IS_REF):
@@ -1505,7 +1552,7 @@
             self._rawobject = capi.C_NULL_OBJECT
 
     def _finalize_(self):
-        if self.flags & INSTANCE_FLAGS_PYTHON_OWNS:
+        if self.rt_flags & INSTANCE_FLAGS_PYTHON_OWNS:
             self.destruct()
 
 W_CPPInstance.typedef = TypeDef(
@@ -1527,31 +1574,33 @@
 
 
 class MemoryRegulator:
-    # TODO: (?) An object address is not unique if e.g. the class has a
-    # public data member of class type at the start of its definition and
-    # has no virtual functions. A _key class that hashes on address and
-    # type would be better, but my attempt failed in the rtyper, claiming
-    # a call on None ("None()") and needed a default ctor. (??)
-    # Note that for now, the associated test carries an m_padding to make
-    # a difference in the addresses.
-    def __init__(self):
-        self.objects = rweakref.RWeakValueDictionary(int, W_CPPInstance)
+    _immutable_ = True
 
-    def register(self, obj):
+    @staticmethod
+    def register(obj):
         if not obj._rawobject:
             return
-        int_address = int(rffi.cast(rffi.LONG, obj._rawobject))
-        self.objects.set(int_address, obj)
+        addr_as_int = int(rffi.cast(rffi.LONG, obj.get_rawobject()))
+        clsdecl = obj.clsdecl
+        assert isinstance(clsdecl, W_CPPClassDecl)
+        clsdecl.cppobjects.set(addr_as_int, obj)
 
-    def unregister(self, obj):
+    @staticmethod
+    def unregister(obj):
         if not obj._rawobject:
             return
-        int_address = int(rffi.cast(rffi.LONG, obj._rawobject))
-        self.objects.set(int_address, None)
+        addr_as_int = int(rffi.cast(rffi.LONG, obj.get_rawobject()))
+        clsdecl = obj.clsdecl
+        assert isinstance(clsdecl, W_CPPClassDecl)
+        clsdecl.cppobjects.set(addr_as_int, None) # actually deletes (pops)
 
-    def retrieve(self, address):
-        int_address = int(rffi.cast(rffi.LONG, address))
-        return self.objects.get(int_address)
+    @staticmethod
+    def retrieve(clsdecl, address):
+        if not address:
+            return None
+        addr_as_int = int(rffi.cast(rffi.LONG, address))
+        assert isinstance(clsdecl, W_CPPClassDecl)
+        return clsdecl.cppobjects.get(addr_as_int)
 
 memory_regulator = MemoryRegulator()
 
@@ -1597,8 +1646,11 @@
 
     # try to recycle existing object if this one is not newly created
     if not fresh and rawobject:
-        obj = memory_regulator.retrieve(rawobject)
-        if obj is not None and obj.clsdecl is clsdecl:
+        address = rawobject
+        if is_ref:
+            address = rffi.cast(capi.C_OBJECT, rffi.cast(rffi.VOIDPP, address)[0])
+        obj = memory_regulator.retrieve(clsdecl, address)
+        if obj is not None:
             return obj
 
     # fresh creation
@@ -1649,7 +1701,7 @@
     """Casts the given instance into an C++-style rvalue."""
     obj = space.interp_w(W_CPPInstance, w_obj)
     if obj:
-        obj.flags |= INSTANCE_FLAGS_IS_RVALUE
+        obj.rt_flags |= INSTANCE_FLAGS_IS_RVALUE
     return w_obj
 
 
diff --git a/pypy/module/_cppyy/lowlevelviews.py b/pypy/module/_cppyy/lowlevelviews.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/_cppyy/lowlevelviews.py
@@ -0,0 +1,54 @@
+# Naked C++ pointers carry no useful size or layout information, but often
+# such information is externnally available. Low level views are arrays with
+# a few more methods allowing such information to be set. Afterwards, it is
+# simple to pass these views on to e.g. numpy (w/o the need to copy).
+
+from pypy.interpreter.error import oefmt
+from pypy.interpreter.gateway import interp2app, unwrap_spec
+from pypy.interpreter.typedef import TypeDef, GetSetProperty, interp_attrproperty_w
+from pypy.module._rawffi.array import W_ArrayInstance
+
+
+class W_LowLevelView(W_ArrayInstance):
+    def __init__(self, space, shape, length, address):
+        assert address   # if not address, base class will allocate memory
+        W_ArrayInstance.__init__(self, space, shape, length, address)
+
+    @unwrap_spec(args_w='args_w')
+    def reshape(self, space, args_w):
+        # llviews are only created from non-zero addresses, so we only need
+        # to adjust length and shape
+
+        nargs = len(args_w)
+        if nargs == 0:
+            raise oefmt(space.w_TypeError, "reshape expects a tuple argument")
+
+        newshape_w = args_w
+        if nargs != 1 or not space.isinstance_w(args_w[0], space.w_tuple) or \
+               not space.len_w(args_w[0]) == 1:
+            raise oefmt(space.w_TypeError,
+                "tuple object of length 1 expected, received %T", args_w[0])
+
+        w_shape = args_w[0]
+
+        # shape in W_ArrayInstance-speak is somewhat different from what
+        # e.g. numpy thinks of it: self.shape contains the info (itemcode,
+        # size, etc.) of a single entry; length is user-facing shape
+        self.length = space.int_w(space.getitem(w_shape, space.newint(0)))
+
+
+W_LowLevelView.typedef = TypeDef(
+    'LowLevelView',
+    __repr__    = interp2app(W_LowLevelView.descr_repr),
+    __setitem__ = interp2app(W_LowLevelView.descr_setitem),
+    __getitem__ = interp2app(W_LowLevelView.descr_getitem),
+    __len__     = interp2app(W_LowLevelView.getlength),
+    buffer      = GetSetProperty(W_LowLevelView.getbuffer),
+    shape       = interp_attrproperty_w('shape', W_LowLevelView),
+    free        = interp2app(W_LowLevelView.free),
+    byptr       = interp2app(W_LowLevelView.byptr),
+    itemaddress = interp2app(W_LowLevelView.descr_itemaddress),
+    reshape     = interp2app(W_LowLevelView.reshape),
+)
+W_ArrayInstance.typedef.acceptable_as_base_class = False
+
diff --git a/pypy/module/_cppyy/pythonify.py b/pypy/module/_cppyy/pythonify.py
--- a/pypy/module/_cppyy/pythonify.py
+++ b/pypy/module/_cppyy/pythonify.py
@@ -73,7 +73,8 @@
 
 # C++ namespace base class (the C++ class base class defined in _post_import_startup)
 class CPPNamespace(with_metaclass(CPPNamespaceMeta, object)):
-     pass
+    def __init__(self):
+        raise TypeError("cannot instantiate namespace '%s'", self.__cppname__)
 
 
 # TODO: this can be moved to the interp level (and share template argument
@@ -450,6 +451,14 @@
             return self.__real_init__(*args)
         pyclass.__init__ = vector_init
 
+        # size-up the return of data()
+        pyclass.__real_data = pyclass.data
+        def data_with_len(self):
+            arr = self.__real_data()
+            arr.reshape((len(self),))
+            return arr
+        pyclass.data = data_with_len
+
     # combine __getitem__ and __len__ to make a pythonized __getitem__
     if '__getitem__' in pyclass.__dict__ and '__len__' in pyclass.__dict__:
         pyclass._getitem__unchecked = pyclass.__getitem__
diff --git a/pypy/module/_cppyy/test/advancedcpp.cxx b/pypy/module/_cppyy/test/advancedcpp.cxx
--- a/pypy/module/_cppyy/test/advancedcpp.cxx
+++ b/pypy/module/_cppyy/test/advancedcpp.cxx
@@ -4,20 +4,32 @@
 
 
 // for testing of default arguments
-#define IMPLEMENT_DEFAULTER_CLASS(type, tname)                               \
+#define IMPLEMENT_DEFAULTERS(type, tname)                                    \
 tname##_defaulter::tname##_defaulter(type a, type b, type c) {               \
-   m_a = a; m_b = b; m_c = c;                                                \
+    m_a = a; m_b = b; m_c = c;                                               \
+}                                                                            \
+type tname##_defaulter_func(int idx, type a, type b, type c) {               \
+    if (idx == 0) return a;                                                  \
+    if (idx == 1) return b;                                                  \
+    if (idx == 2) return c;                                                  \
+    return (type)idx;                                                        \
 }
-IMPLEMENT_DEFAULTER_CLASS(short, short)
-IMPLEMENT_DEFAULTER_CLASS(unsigned short, ushort)
-IMPLEMENT_DEFAULTER_CLASS(int, int)
-IMPLEMENT_DEFAULTER_CLASS(unsigned, uint)
-IMPLEMENT_DEFAULTER_CLASS(long, long)
-IMPLEMENT_DEFAULTER_CLASS(unsigned long, ulong)
-IMPLEMENT_DEFAULTER_CLASS(long long, llong)
-IMPLEMENT_DEFAULTER_CLASS(unsigned long long, ullong)
-IMPLEMENT_DEFAULTER_CLASS(float, float)
-IMPLEMENT_DEFAULTER_CLASS(double, double)
+IMPLEMENT_DEFAULTERS(short, short)
+IMPLEMENT_DEFAULTERS(unsigned short, ushort)
+IMPLEMENT_DEFAULTERS(int, int)
+IMPLEMENT_DEFAULTERS(unsigned, uint)
+IMPLEMENT_DEFAULTERS(long, long)
+IMPLEMENT_DEFAULTERS(unsigned long, ulong)
+IMPLEMENT_DEFAULTERS(long long, llong)
+IMPLEMENT_DEFAULTERS(unsigned long long, ullong)
+IMPLEMENT_DEFAULTERS(float, float)
+IMPLEMENT_DEFAULTERS(double, double)
+
+std::string string_defaulter_func(int idx, const std::string& name1, std::string name2) {
+    if (idx == 0) return name1;
+    if (idx == 1) return name2;
+    return "mies";
+}
 
 
 // for esoteric inheritance testing
@@ -77,11 +89,11 @@
 double my_global_array[500];
 static double sd = 1234.;
 double* my_global_ptr = &sd;
+const char my_global_string2[] = "zus jet teun";
 some_int_holder my_global_int_holders[5] = {
     some_int_holder(13), some_int_holder(42), some_int_holder(88),
     some_int_holder(-1), some_int_holder(17) };
 
-
 // for life-line and identity testing
 int some_class_with_data::some_data::s_num_data = 0;
 
diff --git a/pypy/module/_cppyy/test/advancedcpp.h b/pypy/module/_cppyy/test/advancedcpp.h
--- a/pypy/module/_cppyy/test/advancedcpp.h
+++ b/pypy/module/_cppyy/test/advancedcpp.h
@@ -4,24 +4,27 @@
 
 
 //===========================================================================
-#define DECLARE_DEFAULTER_CLASS(type, tname)                                \
+#define DECLARE_DEFAULTERS(type, tname)                                     \
 class tname##_defaulter {                                                   \
 public:                                                                     \
     tname##_defaulter(type a = 11, type b = 22, type c = 33);               \
                                                                             \
 public:                                                                     \
     type m_a, m_b, m_c;                                                     \
-};
-DECLARE_DEFAULTER_CLASS(short, short)   // for testing of default arguments
-DECLARE_DEFAULTER_CLASS(unsigned short, ushort)
-DECLARE_DEFAULTER_CLASS(int, int)
-DECLARE_DEFAULTER_CLASS(unsigned, uint)
-DECLARE_DEFAULTER_CLASS(long, long)
-DECLARE_DEFAULTER_CLASS(unsigned long, ulong)
-DECLARE_DEFAULTER_CLASS(long long, llong)
-DECLARE_DEFAULTER_CLASS(unsigned long long, ullong)
-DECLARE_DEFAULTER_CLASS(float, float)
-DECLARE_DEFAULTER_CLASS(double, double)
+};                                                                          \
+type tname##_defaulter_func(int idx = 0, type a = 11, type b = 22, type c = 33);
+DECLARE_DEFAULTERS(short, short)   // for testing of default arguments
+DECLARE_DEFAULTERS(unsigned short, ushort)
+DECLARE_DEFAULTERS(int, int)
+DECLARE_DEFAULTERS(unsigned, uint)
+DECLARE_DEFAULTERS(long, long)
+DECLARE_DEFAULTERS(unsigned long, ulong)
+DECLARE_DEFAULTERS(long long, llong)
+DECLARE_DEFAULTERS(unsigned long long, ullong)
+DECLARE_DEFAULTERS(float, float)
+DECLARE_DEFAULTERS(double, double)
+
+std::string string_defaulter_func(int idx, const std::string& name1 = "aap", std::string name2 = "noot");
 
 
 //===========================================================================
@@ -274,7 +277,8 @@
 extern double my_global_double;    // a couple of globals for access testing
 extern double my_global_array[500];
 extern double* my_global_ptr;
-static const char my_global_string[] = "aap " " noot " " mies";
+static const char my_global_string1[] = "aap " " noot " " mies";
+extern const char my_global_string2[];
 
 class some_int_holder {
 public:
@@ -311,6 +315,11 @@
     some_data m_data;
 };
 
+class refers_to_self {             // for data member reuse testing
+public:
+    refers_to_self* m_other = nullptr;
+};
+
 
 //===========================================================================
 class pointer_pass {               // for testing passing of void*'s
@@ -419,3 +428,34 @@
     void throw_anything();
     void throw_exception();
 };
+
+
+//===========================================================================
+class UsingBase {                  // using declaration testing
+public:
+    UsingBase(int n = 13) : m_int(n) {} 
+    virtual char vcheck() { return 'A'; }
+    int m_int;
+};
+
+class UsingDerived : public UsingBase {
+public:
+    using UsingBase::UsingBase;
+    virtual char vcheck() { return 'B'; }
+    int m_int2 = 42;
+};
+
+
+//===========================================================================
+class TypedefToPrivateClass {      // typedef resolution testing
+private:
+    class PC {
+    public:
+        PC(int i) : m_val(i) {}
+        int m_val;
+    };
+
+public:
+    typedef PC PP;
+    PP f() { return PC(42); }
+};
diff --git a/pypy/module/_cppyy/test/advancedcpp.xml b/pypy/module/_cppyy/test/advancedcpp.xml
--- a/pypy/module/_cppyy/test/advancedcpp.xml
+++ b/pypy/module/_cppyy/test/advancedcpp.xml
@@ -1,6 +1,7 @@
 <lcgdict>
 
   <class pattern="*_defaulter" />
+  <function pattern="*_defaulter_func" />
 
   <class name="base_class" />
   <class name="derived_class" />
@@ -32,6 +33,7 @@
   <class name="some_convertible" />
   <class name="some_class_with_data" />
   <class name="some_class_with_data::some_data" />
+  <class name="refers_to_self" />
 
   <class name="some_comparable" />
   <function name="operator==" />
@@ -59,5 +61,7 @@
   <class name="overload_the_other_way" />
 
   <class name="Thrower" />
+  <class pattern="Using*" />
+  <class name="TypedefToPrivateClass" />
 
 </lcgdict>
diff --git a/pypy/module/_cppyy/test/stltypes.cxx b/pypy/module/_cppyy/test/stltypes.cxx
--- a/pypy/module/_cppyy/test/stltypes.cxx
+++ b/pypy/module/_cppyy/test/stltypes.cxx
@@ -1,6 +1,20 @@
 #include "stltypes.h"
 
 
+//- explicit instantiations of used comparisons
+#if defined __clang__
+namespace std {
+#define ns_prefix std::
+#elif defined(__GNUC__) || defined(__GNUG__)
+namespace __gnu_cxx {
+#define ns_prefix
+#endif
+template bool ns_prefix operator==(const std::vector<int>::iterator&,
+                         const std::vector<int>::iterator&);
+template bool ns_prefix operator!=(const std::vector<int>::iterator&,
+                         const std::vector<int>::iterator&);
+}
+
 //- class with lots of std::string handling
 stringy_class::stringy_class(const char* s) : m_string(s) {}
 
@@ -9,3 +23,33 @@
 
 void stringy_class::set_string1(const std::string& s) { m_string = s; }
 void stringy_class::set_string2(std::string s) { m_string = s; }
+
+
+//- helpers for testing array
+int ArrayTest::get_pp_px(Point** p, int idx) {
+    return p[idx]->px;
+}
+
+int ArrayTest::get_pp_py(Point** p, int idx) {
+    return p[idx]->py;
+}
+
+int ArrayTest::get_pa_px(Point* p[], int idx) {
+    return p[idx]->px;
+}
+
+int ArrayTest::get_pa_py(Point* p[], int idx) {
+    return p[idx]->py;
+}
+
+
+// helpers for string testing
+std::string str_array_1[3] = {"a", "b", "c"};
+std::string str_array_2[]  = {"d", "e", "f", "g"};
+std::string str_array_3[3][2] = {{"a", "b"}, {"c", "d"}, {"e", "f"}};
+std::string str_array_4[4][2][2] = {
+     {{"a", "b"}, {"c", "d"}},
+     {{"e", "f"}, {"g", "h"}},
+     {{"i", "j"}, {"k", "l"}},
+     {{"m", "n"}, {"o", "p"}},
+};
diff --git a/pypy/module/_cppyy/test/stltypes.h b/pypy/module/_cppyy/test/stltypes.h
--- a/pypy/module/_cppyy/test/stltypes.h
+++ b/pypy/module/_cppyy/test/stltypes.h
@@ -37,3 +37,49 @@
    std::string operator[](double) { return "double"; }
    std::string operator[](const std::string&) { return "string"; }
 };      
+
+
+//- instantiations of used STL types
+namespace {
+
+    stl_like_class<int> stlc_1;
+
+} // unnamed namespace
+
+
+// comps for int only to allow testing: normal use of vector is looping over a
+// range-checked version of __getitem__
+#if defined(__clang__) && defined(__APPLE__)
+namespace std {
+#define ns_prefix std::
+#elif defined(__GNUC__) || defined(__GNUG__)
+namespace __gnu_cxx {
+#define ns_prefix
+#endif
+extern template bool ns_prefix operator==(const std::vector<int>::iterator&,
+                         const std::vector<int>::iterator&);
+extern template bool ns_prefix operator!=(const std::vector<int>::iterator&,
+                         const std::vector<int>::iterator&);
+}
+
+
+//- helpers for testing array
+namespace ArrayTest {
+
+struct Point {
+    int px, py;
+};
+
+int get_pp_px(Point** p, int idx);
+int get_pp_py(Point** p, int idx);
+int get_pa_px(Point* p[], int idx);
+int get_pa_py(Point* p[], int idx);
+
+} // namespace ArrayTest
+
+
+// helpers for string testing
+extern std::string str_array_1[3];
+extern std::string str_array_2[];
+extern std::string str_array_3[3][2];
+extern std::string str_array_4[4][2][2];
diff --git a/pypy/module/_cppyy/test/stltypes.xml b/pypy/module/_cppyy/test/stltypes.xml
--- a/pypy/module/_cppyy/test/stltypes.xml
+++ b/pypy/module/_cppyy/test/stltypes.xml
@@ -1,10 +1,13 @@
 <lcgdict>
 
-  <namespace name="std" />
+  <class name="just_a_class" />
+  <class name="stringy_class" />
+  <class pattern="stl_like_class<*>" />
 
-  <class name="just_a_class" />
+  <namespace name="ArrayTest" />
+  <class pattern="ArrayTest::*" />
+  <function pattern="ArrayTest::*" />
 
-  <class name="std::string" />
-  <class name="stringy_class" />
+  <variable pattern="str_array_*" />
 
 </lcgdict>
diff --git a/pypy/module/_cppyy/test/test_advancedcpp.py b/pypy/module/_cppyy/test/test_advancedcpp.py
--- a/pypy/module/_cppyy/test/test_advancedcpp.py
+++ b/pypy/module/_cppyy/test/test_advancedcpp.py
@@ -56,6 +56,12 @@
             assert d.m_b ==  t(4)
             assert d.m_c ==  t(5)
             d.__destruct__()
+
+            defaulter_func = getattr(cppyy.gbl, '%s_defaulter_func' %n)
+            answers = [11, 22, 33, 3]
+            for idx in range(4):
+                assert defaulter_func(idx) == answers[idx]
+
         test_defaulter('short',  int)
         test_defaulter('ushort', int)
         test_defaulter('int',    int)
@@ -67,6 +73,13 @@
         test_defaulter('float',  float)
         test_defaulter('double', float)
 
+        assert cppyy.gbl.string_defaulter_func(0)               == "aap"
+        assert cppyy.gbl.string_defaulter_func(0, "zus")        == "zus"
+        assert cppyy.gbl.string_defaulter_func(1)               == "noot"
+        assert cppyy.gbl.string_defaulter_func(1, "zus")        == "noot"
+        assert cppyy.gbl.string_defaulter_func(1, "zus", "jet") == "jet"
+        assert cppyy.gbl.string_defaulter_func(2)               == "mies"
+
     def test02_simple_inheritance(self):
         """Test binding of a basic inheritance structure"""
 
@@ -148,6 +161,8 @@
         assert gbl.a_ns.d_ns.e_class.f_class.s_f      == 66
         assert gbl.a_ns.d_ns.e_class.f_class().m_f    == -6
 
+        raises(TypeError, gbl.a_ns)
+
     def test03a_namespace_lookup_on_update(self):
         """Test whether namespaces can be shared across dictionaries."""
 
@@ -474,6 +489,23 @@
         d2.__destruct__()
         d1.__destruct__()
 
+        RTS = cppyy.gbl.refers_to_self
+
+        r1 = RTS()
+        r2 = RTS()
+        r1.m_other = r2
+
+        r3 = r1.m_other
+        r4 = r1.m_other
+        assert r3 is r4
+
+        assert r3 == r2
+        assert r3 is r2
+
+        r3.extra = 42
+        assert r2.extra == 42
+        assert r4.extra == 42
+
     def test11_multi_methods(self):
         """Test calling of methods from multiple inheritance"""
 
@@ -653,10 +685,18 @@
 
         assert cppyy.gbl.my_global_double == 12.
         assert len(cppyy.gbl.my_global_array) == 500
-        assert cppyy.gbl.my_global_string == "aap  noot  mies"
+        assert cppyy.gbl.my_global_string1 == "aap  noot  mies"
+        return     # next line currently crashes
+        assert cppyy.gbl.my_global_string2 == "zus jet teun"
         # TODO: currently fails b/c double** not understood as &double*
         #assert cppyy.gbl.my_global_ptr[0] == 1234.
 
+        v = cppyy.gbl.my_global_int_holders
+        assert len(v) == 5
+        expected_vals = [13, 42, 88, -1, 17]
+        for i in range(len(v)):
+            assert v[i].m_val == expected_vals[i]
+
     def test22_exceptions(self):
         """Catching of C++ exceptions"""
 
@@ -675,3 +715,35 @@
             t.throw_exception()
         except Exception as e:
             "C++ function failed" in str(e)
+
+    def test23_using(self):
+        """Accessibility of using declarations"""
+
+        import _cppyy as cppyy
+
+        assert cppyy.gbl.UsingBase().vcheck() == 'A'
+
+        B = cppyy.gbl.UsingDerived
+        assert not 'UsingBase' in B.__init__.__doc__
+
+        b1 = B()
+        assert b1.m_int    == 13
+        assert b1.m_int2   == 42
+        assert b1.vcheck() == 'B'
+ 
+        b2 = B(10)
+        assert b2.m_int    == 10
+        assert b2.m_int2   == 42
+        assert b2.vcheck() == 'B'
+
+        b3 = B(b2)
+        assert b3.m_int    == 10
+        assert b3.m_int2   == 42
+        assert b3.vcheck() == 'B'
+
+    def test24_typedef_to_private_class(self):
+        """Typedefs to private classes should not resolve"""
+
+        import _cppyy as cppyy
+
+        assert cppyy.gbl.TypedefToPrivateClass().f().m_val == 42
diff --git a/pypy/module/_cppyy/test/test_datatypes.py b/pypy/module/_cppyy/test/test_datatypes.py
--- a/pypy/module/_cppyy/test/test_datatypes.py
+++ b/pypy/module/_cppyy/test/test_datatypes.py
@@ -183,7 +183,7 @@
         names = ['uchar', 'short', 'ushort', 'int', 'uint', 'long', 'ulong']
         import array
         a = range(self.N)
-        atypes = ['B', 'h', 'H', 'i', 'I', 'l', 'L' ]
+        atypes = ['B', 'h', 'H', 'i', 'I', 'l', 'L']
         for j in range(len(names)):
             b = array.array(atypes[j], a)
             setattr(c, 'm_'+names[j]+'_array', b)     # buffer copies
@@ -191,6 +191,7 @@
                 assert eval('c.m_%s_array[i]' % names[j]) == b[i]
 
             setattr(c, 'm_'+names[j]+'_array2', b)    # pointer copies
+            assert 3 < self.N
             b[3] = 28
             for i in range(self.N):
                 assert eval('c.m_%s_array2[i]' % names[j]) == b[i]
@@ -212,7 +213,7 @@
 
         a = range(self.N)
         # test arrays in mixed order, to give overload resolution a workout
-        for t in ['d', 'i', 'f', 'H', 'I', 'h', 'L', 'l' ]:
+        for t in ['d', 'i', 'f', 'H', 'I', 'h', 'L', 'l']:
             b = array.array(t, a)
 
             # typed passing
@@ -691,9 +692,6 @@
                      'get_long_array',   'get_long_array2',
                      'get_ulong_array',  'get_ulong_array2']:
             arr = getattr(c, func)()
-            arr = arr.shape.fromaddress(arr.itemaddress(0), self.N)
-
-            """TODO: interface change ...
             arr.reshape((self.N,))
             assert len(arr) == self.N
 
@@ -705,7 +703,7 @@
 
             l = list(arr)
             for i in range(self.N):
-                assert arr[i] == l[i]"""
+                assert arr[i] == l[i]
 
     def test20_voidp(self):
         """Test usage of void* data"""
@@ -758,6 +756,15 @@
     def test21_function_pointers(self):
         """Function pointer passing"""
 
+        import os
+
+        # TODO: currently crashes if fast path disabled
+        try:
+            if os.environ['CPPYY_DISABLE_FASTPATH']:
+                return
+        except KeyError:
+            pass
+
         import _cppyy as cppyy
 
         f1 = cppyy.gbl.sum_of_int
diff --git a/pypy/module/_cppyy/test/test_overloads.py b/pypy/module/_cppyy/test/test_overloads.py
--- a/pypy/module/_cppyy/test/test_overloads.py
+++ b/pypy/module/_cppyy/test/test_overloads.py
@@ -58,15 +58,19 @@
         c = c_overload()
         raises(TypeError, c.__dispatch__, 'get_int', 12)
         raises(LookupError, c.__dispatch__, 'get_int', 'does_not_exist')
-        assert c.__dispatch__('get_int', 'a_overload*')(a_overload()) == 42
-        assert c.__dispatch__('get_int', 'b_overload*')(b_overload()) == 13
+        assert c.__dispatch__('get_int', 'a_overload*')(a_overload())             == 42
+        assert c_overload.get_int.__overload__('a_overload*')(c, a_overload())    == 42
+        assert c.__dispatch__('get_int', 'b_overload*')(b_overload())             == 13
+        assert c_overload.get_int.__overload__('b_overload*')(c, b_overload())    == 13
 
         assert c_overload().__dispatch__('get_int', 'a_overload*')(a_overload())  == 42
         # TODO: #assert c_overload.__dispatch__('get_int', 'b_overload*')(c, b_overload()) == 13
 
         d = d_overload()
-        assert d.__dispatch__('get_int', 'a_overload*')(a_overload()) == 42
-        assert d.__dispatch__('get_int', 'b_overload*')(b_overload()) == 13
+        assert d.__dispatch__('get_int', 'a_overload*')(a_overload())             == 42
+        assert d_overload.get_int.__overload__('a_overload*')(d, a_overload())    == 42
+        assert d.__dispatch__('get_int', 'b_overload*')(b_overload())             == 13
+        assert d_overload.get_int.__overload__('b_overload*')(d, b_overload())    == 13
 
         nb = ns_a_overload.b_overload()
         raises(TypeError, nb.f, c_overload())
diff --git a/pypy/module/_cppyy/test/test_stltypes.py b/pypy/module/_cppyy/test/test_stltypes.py
--- a/pypy/module/_cppyy/test/test_stltypes.py
+++ b/pypy/module/_cppyy/test/test_stltypes.py
@@ -22,12 +22,12 @@
     def test01_builtin_type_vector_types(self):
         """Test access to std::vector<int>/std::vector<double>"""
 
-        import _cppyy
+        import _cppyy as cppyy
 
-        assert _cppyy.gbl.std        is _cppyy.gbl.std
-        assert _cppyy.gbl.std.vector is _cppyy.gbl.std.vector
+        assert cppyy.gbl.std        is cppyy.gbl.std
+        assert cppyy.gbl.std.vector is cppyy.gbl.std.vector
 
-        assert callable(_cppyy.gbl.std.vector)
+        assert callable(cppyy.gbl.std.vector)
 
         type_info = (
             ("int",     int),
@@ -36,14 +36,14 @@
         )
 
         for c_type, p_type in type_info:
-            tv1 = getattr(_cppyy.gbl.std, 'vector<%s>' % c_type)
-            tv2 = _cppyy.gbl.std.vector(p_type)
+            tv1 = getattr(cppyy.gbl.std, 'vector<%s>' % c_type)
+            tv2 = cppyy.gbl.std.vector(p_type)
             assert tv1 is tv2
-            assert tv1.iterator is _cppyy.gbl.std.vector(p_type).iterator
+            assert tv1.iterator is cppyy.gbl.std.vector(p_type).iterator
 
             #----- 
-            v = tv1(); v += range(self.N)    # default args from Reflex are useless :/
-            if p_type == int:                # only type with == and != reflected in .xml
+            v = tv1(); v += range(self.N)
+            if p_type == int:
                 assert v.begin().__eq__(v.begin())
                 assert v.begin() == v.begin()
                 assert v.end() == v.end()
@@ -58,6 +58,7 @@
 
             assert v.size() == self.N
             assert len(v) == self.N
+            assert len(v.data()) == self.N
 
             #-----
             v = tv1()
@@ -73,16 +74,16 @@
     def test02_user_type_vector_type(self):
         """Test access to an std::vector<just_a_class>"""
 
-        import _cppyy
+        import _cppyy as cppyy
 
-        assert _cppyy.gbl.std        is _cppyy.gbl.std
-        assert _cppyy.gbl.std.vector is _cppyy.gbl.std.vector
+        assert cppyy.gbl.std        is cppyy.gbl.std
+        assert cppyy.gbl.std.vector is cppyy.gbl.std.vector
 
-        assert callable(_cppyy.gbl.std.vector)
+        assert callable(cppyy.gbl.std.vector)
 
-        tv1 = getattr(_cppyy.gbl.std, 'vector<just_a_class>')
-        tv2 = _cppyy.gbl.std.vector('just_a_class')
-        tv3 = _cppyy.gbl.std.vector(_cppyy.gbl.just_a_class)
+        tv1 = getattr(cppyy.gbl.std, 'vector<just_a_class>')
+        tv2 = cppyy.gbl.std.vector('just_a_class')
+        tv3 = cppyy.gbl.std.vector(cppyy.gbl.just_a_class)
 
         assert tv1 is tv2
         assert tv2 is tv3
@@ -95,7 +96,7 @@
         assert hasattr(v, 'end' )
 
         for i in range(self.N):
-            v.push_back(_cppyy.gbl.just_a_class())
+            v.push_back(cppyy.gbl.just_a_class())
             v[i].m_i = i
             assert v[i].m_i == i
 
@@ -105,9 +106,9 @@
     def test03_empty_vector_type(self):
         """Test behavior of empty std::vector<int>"""
 
-        import _cppyy
+        import _cppyy as cppyy
 
-        v = _cppyy.gbl.std.vector(int)()
+        v = cppyy.gbl.std.vector(int)()
         for arg in v:
             pass
         v.__destruct__()
@@ -115,9 +116,9 @@
     def test04_vector_iteration(self):
         """Test iteration over an std::vector<int>"""
 
-        import _cppyy
+        import _cppyy as cppyy
 
-        v = _cppyy.gbl.std.vector(int)()
+        v = cppyy.gbl.std.vector(int)()
 
         for i in range(self.N):
             v.push_back(i)
@@ -140,9 +141,9 @@
     def test05_push_back_iterables_with_iadd(self):
         """Test usage of += of iterable on push_back-able container"""
 
-        import _cppyy
+        import _cppyy as cppyy
 
-        v = _cppyy.gbl.std.vector(int)()
+        v = cppyy.gbl.std.vector(int)()
 
         v += [1, 2, 3]
         assert len(v) == 3
@@ -159,7 +160,7 @@
         raises(TypeError, v.__iadd__, (7, '8'))  # string shouldn't pass
         assert len(v) == 7   # TODO: decide whether this should roll-back
 
-        v2 = _cppyy.gbl.std.vector(int)()
+        v2 = cppyy.gbl.std.vector(int)()
         v2 += [8, 9]
         assert len(v2) == 2
         assert v2[0] == 8
@@ -174,9 +175,9 @@
     def test06_vector_indexing(self):
         """Test python-style indexing to an std::vector<int>"""
 
-        import _cppyy
+        import _cppyy as cppyy
 
-        v = _cppyy.gbl.std.vector(int)()
+        v = cppyy.gbl.std.vector(int)()
 
         for i in range(self.N):
             v.push_back(i)
@@ -209,9 +210,9 @@
     def test01_string_argument_passing(self):
         """Test mapping of python strings and std::string"""
 
-        import _cppyy
-        std = _cppyy.gbl.std
-        stringy_class = _cppyy.gbl.stringy_class
+        import _cppyy as cppyy
+        std = cppyy.gbl.std
+        stringy_class = cppyy.gbl.stringy_class
 
         c, s = stringy_class(""), std.string("test1")
 
@@ -240,9 +241,9 @@
     def test02_string_data_access(self):
         """Test access to std::string object data members"""
 
-        import _cppyy
-        std = _cppyy.gbl.std
-        stringy_class = _cppyy.gbl.stringy_class
+        import _cppyy as cppyy
+        std = cppyy.gbl.std
+        stringy_class = cppyy.gbl.stringy_class
 
         c, s = stringy_class("dummy"), std.string("test string")
 
@@ -261,9 +262,9 @@
 
         return # don't bother; is fixed in cling-support
 
-        import _cppyy
-        std = _cppyy.gbl.std
-        stringy_class = _cppyy.gbl.stringy_class
+        import _cppyy as cppyy
+        std = cppyy.gbl.std
+        stringy_class = cppyy.gbl.stringy_class
 
         t0 = "aap\0noot"
         assert t0 == "aap\0noot"
@@ -288,8 +289,8 @@
     def test01_builtin_list_type(self):
         """Test access to a list<int>"""
 
-        import _cppyy
-        std = _cppyy.gbl.std
+        import _cppyy as cppyy
+        std = cppyy.gbl.std
 
         type_info = (
             ("int",     int),
@@ -299,9 +300,9 @@
 
         for c_type, p_type in type_info:
             tl1 = getattr(std, 'list<%s>' % c_type)
-            tl2 = _cppyy.gbl.std.list(p_type)
+            tl2 = cppyy.gbl.std.list(p_type)
             assert tl1 is tl2
-            assert tl1.iterator is _cppyy.gbl.std.list(p_type).iterator
+            assert tl1.iterator is cppyy.gbl.std.list(p_type).iterator
 
             #-----
             a = tl1()
@@ -323,8 +324,8 @@
     def test02_empty_list_type(self):
         """Test behavior of empty list<int>"""
 
-        import _cppyy
-        std = _cppyy.gbl.std
+        import _cppyy as cppyy
+        std = cppyy.gbl.std
 
         a = std.list(int)()
         for arg in a:
@@ -344,8 +345,8 @@
     def test01_builtin_map_type(self):
         """Test access to a map<int,int>"""
 
-        import _cppyy
-        std = _cppyy.gbl.std
+        import _cppyy as cppyy
+        std = cppyy.gbl.std
 
         a = std.map(int, int)()
         for i in range(self.N):
@@ -373,8 +374,8 @@
     def test02_keyed_maptype(self):
         """Test access to a map<std::string,int>"""
 
-        import _cppyy
-        std = _cppyy.gbl.std
+        import _cppyy as cppyy
+        std = cppyy.gbl.std
 
         a = std.map(std.string, int)()
         for i in range(self.N):
@@ -386,8 +387,8 @@
     def test03_empty_maptype(self):
         """Test behavior of empty map<int,int>"""
 
-        import _cppyy
-        std = _cppyy.gbl.std
+        import _cppyy as cppyy
+        std = cppyy.gbl.std
 
         m = std.map(int, int)()
         for key, value in m:
@@ -396,8 +397,9 @@
     def test04_unsignedvalue_typemap_types(self):
         """Test assignability of maps with unsigned value types"""
 
-        import _cppyy, math, sys
-        std = _cppyy.gbl.std
+        import _cppyy as cppyy
+        import math, sys
+        std = cppyy.gbl.std
 
         mui = std.map(str, 'unsigned int')()
         mui['one'] = 1
@@ -420,8 +422,8 @@
     def test05_STL_like_class_indexing_overloads(self):
         """Test overloading of operator[] in STL like class"""
 
-        import _cppyy
-        stl_like_class = _cppyy.gbl.stl_like_class
+        import _cppyy as cppyy
+        stl_like_class = cppyy.gbl.stl_like_class
 
         a = stl_like_class(int)()
         assert a["some string" ] == 'string'
@@ -430,8 +432,8 @@
     def test06_STL_like_class_iterators(self):
         """Test the iterator protocol mapping for an STL like class"""
 
-        import _cppyy
-        stl_like_class = _cppyy.gbl.stl_like_class
+        import _cppyy as cppyy
+        stl_like_class = cppyy.gbl.stl_like_class
 
         a = stl_like_class(int)()
         for i in a:
@@ -452,8 +454,8 @@
     def test01_builtin_vector_iterators(self):
         """Test iterator comparison with operator== reflected"""
 
-        import _cppyy
-        std = _cppyy.gbl.std
+        import _cppyy as cppyy
+        std = cppyy.gbl.std
 
         v = std.vector(int)()
         v.resize(1)
@@ -489,9 +491,9 @@
     def test01_explicit_templates(self):
         """Explicit use of Template class"""
 
-        import _cppyy
+        import _cppyy as cppyy
 
-        vector = _cppyy.Template('vector', _cppyy.gbl.std)
+        vector = cppyy.Template('vector', cppyy.gbl.std)
         assert vector[int] == vector(int)
 
         v = vector[int]()
@@ -501,3 +503,54 @@
         assert len(v) == N
         for i in range(N):
             assert v[i] == i
+
+
+class AppTestSTLARRAY:
+    spaceconfig = dict(usemodules=['_cppyy', '_rawffi', 'itertools'])
+
+    def setup_class(cls):
+        cls.w_test_dct  = cls.space.newtext(test_dct)
+        cls.w_stlarray = cls.space.appexec([], """():
+            import ctypes, _cppyy
+            _cppyy._post_import_startup()
+            return ctypes.CDLL(%r, ctypes.RTLD_GLOBAL)""" % (test_dct, ))
+
+    def test01_array_of_basic_types(self):
+        """Usage of std::array of basic types"""
+
+        import _cppyy as cppyy
+        std = cppyy.gbl.std
+
+        a = std.array[int, 4]()
+        assert len(a) == 4
+        for i in range(len(a)):
+            a[i] = i
+            assert a[i] == i
+
+    def test02_array_of_pods(self):
+        """Usage of std::array of PODs"""
+
+        import _cppyy as cppyy
+        gbl, std = cppyy.gbl, cppyy.gbl.std
+
+        a = std.array[gbl.ArrayTest.Point, 4]()
+        assert len(a) == 4
+        for i in range(len(a)):
+            a[i].px = i
+            assert a[i].px == i
+            a[i].py = i**2
+            assert a[i].py == i**2
+
+    def test03_array_of_pointer_to_pods(self):
+        """Usage of std::array of pointer to PODs"""
+
+        import cppyy
+        from cppyy import gbl
+        from cppyy.gbl import std
+
+        ll = [gbl.ArrayTest.Point() for i in range(4)]
+        for i in range(len(ll)):
+            ll[i].px = 13*i
+            ll[i].py = 42*i
+
+        # more tests in cppyy/test/test_stltypes.py, but currently not supported
diff --git a/pypy/module/_multiprocessing/interp_semaphore.py b/pypy/module/_multiprocessing/interp_semaphore.py
--- a/pypy/module/_multiprocessing/interp_semaphore.py
+++ b/pypy/module/_multiprocessing/interp_semaphore.py
@@ -60,7 +60,7 @@
         TIMESPEC = platform.Struct('struct timespec', [('tv_sec', rffi.TIME_T),
                                                        ('tv_nsec', rffi.LONG)])
         SEM_FAILED = platform.ConstantInteger('SEM_FAILED')
-        SEM_VALUE_MAX = platform.ConstantInteger('SEM_VALUE_MAX')
+        SEM_VALUE_MAX = platform.DefinedConstantInteger('SEM_VALUE_MAX')
         SEM_TIMED_WAIT = platform.Has('sem_timedwait')
         SEM_T_SIZE = platform.SizeOf('sem_t')
 
@@ -73,6 +73,8 @@
     #                rffi.cast(SEM_T, config['SEM_FAILED'])
     SEM_FAILED     = config['SEM_FAILED']
     SEM_VALUE_MAX  = config['SEM_VALUE_MAX']
+    if SEM_VALUE_MAX is None:      # on Hurd
+        SEM_VALUE_MAX = sys.maxint
     SEM_TIMED_WAIT = config['SEM_TIMED_WAIT']
     SEM_T_SIZE = config['SEM_T_SIZE']
     if sys.platform == 'darwin':
diff --git a/pypy/module/cpyext/cdatetime.py b/pypy/module/cpyext/cdatetime.py
--- a/pypy/module/cpyext/cdatetime.py
+++ b/pypy/module/cpyext/cdatetime.py
@@ -12,18 +12,19 @@
 from pypy.module.__pypy__.interp_pypydatetime import (W_DateTime_Date,
     W_DateTime_Time, W_DateTime_Delta)
 from rpython.tool.sourcetools import func_renamer
+from pypy.module.cpyext.state import State
 
 cts.parse_header(parse_dir / 'cpyext_datetime.h')
 
 
 PyDateTime_CAPI = cts.gettype('PyDateTime_CAPI')
 
-datetimeAPI_global = []
 
 @cpython_api([], lltype.Ptr(PyDateTime_CAPI))
 def _PyDateTime_Import(space):
-    if len(datetimeAPI_global) >0:
-        return datetimeAPI_global[0]
+    state = space.fromcache(State)
+    if len(state.datetimeAPI) > 0:
+        return state.datetimeAPI[0]
     datetimeAPI = lltype.malloc(PyDateTime_CAPI, flavor='raw',
                                 track_allocation=False)
 
@@ -66,8 +67,8 @@
         _PyDelta_FromDelta.api_func.functype,
         _PyDelta_FromDelta.api_func.get_wrapper(space))
 
-    datetimeAPI_global.append(datetimeAPI)
-    return datetimeAPI
+    state.datetimeAPI.append(datetimeAPI)
+    return state.datetimeAPI[0]
 
 PyDateTime_Time = cts.gettype('PyDateTime_Time*')
 PyDateTime_DateTime = cts.gettype('PyDateTime_DateTime*')
@@ -135,8 +136,10 @@
     '''Fills a newly allocated py_obj from the w_obj
     If it is a datetime.time or datetime.datetime, it may have tzinfo
     '''
-    assert len(datetimeAPI_global) > 0
-    if datetimeAPI_global[0].c_TimeType == py_obj.c_ob_type:
+    state = space.fromcache(State)
+    # cannot raise here, so just crash
+    assert len(state.datetimeAPI) > 0
+    if state.datetimeAPI[0].c_TimeType == py_obj.c_ob_type:
         py_datetime = rffi.cast(PyDateTime_Time, py_obj)
         w_tzinfo = space.getattr(w_obj, space.newtext('tzinfo'))
         if space.is_none(w_tzinfo):
@@ -145,7 +148,7 @@
         else:
             py_datetime.c_hastzinfo = cts.cast('unsigned char', 1)
             py_datetime.c_tzinfo = make_ref(space, w_tzinfo)
-    elif datetimeAPI_global[0].c_DateTimeType == py_obj.c_ob_type:
+    elif state.datetimeAPI[0].c_DateTimeType == py_obj.c_ob_type:
         # For now this is exactly the same structure as PyDateTime_Time
         py_datetime = rffi.cast(PyDateTime_DateTime, py_obj)
         w_tzinfo = space.getattr(w_obj, space.newtext('tzinfo'))
@@ -159,12 +162,14 @@
 @slot_function([PyObject], lltype.Void)
 def type_dealloc(space, py_obj):
     from pypy.module.cpyext.object import _dealloc
-    assert len(datetimeAPI_global) > 0
-    if datetimeAPI_global[0].c_TimeType == py_obj.c_ob_type:
+    state = space.fromcache(State)
+    # cannot raise here, so just crash
+    assert len(state.datetimeAPI) > 0
+    if state.datetimeAPI[0].c_TimeType == py_obj.c_ob_type:
         py_datetime = rffi.cast(PyDateTime_Time, py_obj)
         if (widen(py_datetime.c_hastzinfo) != 0):
             decref(space, py_datetime.c_tzinfo)
-    elif datetimeAPI_global[0].c_DateTimeType == py_obj.c_ob_type:
+    elif state.datetimeAPI[0].c_DateTimeType == py_obj.c_ob_type:
         py_datetime = rffi.cast(PyDateTime_DateTime, py_obj)
         if (widen(py_datetime.c_hastzinfo) != 0):
             decref(space, py_datetime.c_tzinfo)
diff --git a/pypy/module/cpyext/state.py b/pypy/module/cpyext/state.py
--- a/pypy/module/cpyext/state.py
+++ b/pypy/module/cpyext/state.py
@@ -42,6 +42,8 @@
         # A mapping {filename: copy-of-the-w_dict}, similar to CPython's
         # variable 'extensions' in Python/import.c.
         self.extensions = {}
+        # XXX will leak if _PyDateTime_Import already called
+        self.datetimeAPI = []
 
     def set_exception(self, operror):
         self.clear_exception()
diff --git a/rpython/jit/backend/detect_cpu.py b/rpython/jit/backend/detect_cpu.py
--- a/rpython/jit/backend/detect_cpu.py
+++ b/rpython/jit/backend/detect_cpu.py
@@ -57,6 +57,7 @@
             'i486': MODEL_X86,
             'i586': MODEL_X86,
             'i686': MODEL_X86,
+            'i686-AT386': MODEL_X86,  # Hurd
             'i86pc': MODEL_X86,    # Solaris/Intel
             'x86': MODEL_X86,      # Apple
             'Power Macintosh': MODEL_PPC_64,
diff --git a/rpython/rlib/buffer.py b/rpython/rlib/buffer.py
--- a/rpython/rlib/buffer.py
+++ b/rpython/rlib/buffer.py
@@ -6,7 +6,7 @@
 from rpython.rtyper.lltypesystem.rstr import STR
 from rpython.rtyper.lltypesystem.rlist import LIST_OF
 from rpython.rtyper.annlowlevel import llstr
-from rpython.rlib.objectmodel import specialize
+from rpython.rlib.objectmodel import specialize, we_are_translated
 from rpython.rlib import jit
 from rpython.rlib.rgc import (resizable_list_supporting_raw_ptr,
                               nonmoving_raw_ptr_for_resizable_list,
@@ -114,6 +114,12 @@
         """
         raise CannotWrite
 
+    def get_raw_address(self):
+        msg = "cannot take the raw address of this buffer"
+        if not we_are_translated():
+            msg += " '%s'" % (self,)
+        raise ValueError(msg)
+
 
 class RawBuffer(Buffer):
     """
diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py
--- a/rpython/rlib/rposix.py
+++ b/rpython/rlib/rposix.py
@@ -1086,9 +1086,12 @@
         else:
             return bool(c_func(status))
 
-WAIT_MACROS = ['WCOREDUMP', 'WIFCONTINUED', 'WIFSTOPPED',
+WAIT_MACROS = ['WCOREDUMP', 'WIFSTOPPED',
                'WIFSIGNALED', 'WIFEXITED',
                'WEXITSTATUS', 'WSTOPSIG', 'WTERMSIG']
+if not sys.platform.startswith('gnu'):
+    WAIT_MACROS.append('WIFCONTINUED')
+
 for name in WAIT_MACROS:
     _make_waitmacro(name)
 
diff --git a/rpython/rlib/rvmprof/cintf.py b/rpython/rlib/rvmprof/cintf.py
--- a/rpython/rlib/rvmprof/cintf.py
+++ b/rpython/rlib/rvmprof/cintf.py
@@ -17,7 +17,7 @@
 
 # vmprof works only on x86 for now
 IS_SUPPORTED = False
-if sys.platform in ('darwin', 'linux', 'linux2'):
+if sys.platform in ('darwin', 'linux', 'linux2') or sys.platform.startswith('freebsd'):
     try:
         IS_SUPPORTED = detect_cpu.autodetect().startswith('x86')
     except detect_cpu.ProcessorAutodetectError:
diff --git a/rpython/rlib/rvmprof/dummy.py b/rpython/rlib/rvmprof/dummy.py
--- a/rpython/rlib/rvmprof/dummy.py
+++ b/rpython/rlib/rvmprof/dummy.py
@@ -1,6 +1,7 @@
 from rpython.rlib.objectmodel import specialize
 
 class DummyVMProf(object):
+    is_enabled = False
 
     def __init__(self):
         self._unique_id = 0
diff --git a/rpython/rlib/rvmprof/rvmprof.py b/rpython/rlib/rvmprof/rvmprof.py
--- a/rpython/rlib/rvmprof/rvmprof.py
+++ b/rpython/rlib/rvmprof/rvmprof.py
@@ -23,6 +23,7 @@
 VMPROF_GC_TAG = 5
 
 class VMProfError(Exception):
+    msg = ''   # annotation hack
     def __init__(self, msg):
         self.msg = msg
     def __str__(self):
diff --git a/rpython/rlib/rvmprof/src/shared/vmprof_unix.h b/rpython/rlib/rvmprof/src/shared/vmprof_unix.h
--- a/rpython/rlib/rvmprof/src/shared/vmprof_unix.h
+++ b/rpython/rlib/rvmprof/src/shared/vmprof_unix.h
@@ -24,6 +24,9 @@
 
 #include "vmprof_mt.h"
 
+#ifdef __FreeBSD__
+#include <ucontext.h>
+#endif
 #include <signal.h>
 
 RPY_EXTERN void vmprof_ignore_signals(int ignored);
diff --git a/rpython/rlib/rvmprof/traceback.py b/rpython/rlib/rvmprof/traceback.py
--- a/rpython/rlib/rvmprof/traceback.py
+++ b/rpython/rlib/rvmprof/traceback.py
@@ -13,6 +13,8 @@
     array_length).  The caller must free array_p.  Not for signal handlers:
     for these, call vmprof_get_traceback() from C code.
     """
+    if not cintf.IS_SUPPORTED:
+        return (None, 0)
     _cintf = rvmprof._get_vmprof().cintf
     size = estimate_number_of_entries * 2 + 4
     stack = cintf.get_rvmprof_stack()
@@ -47,6 +49,8 @@
     'code_obj' may be None if it can't be determined.  'loc' is one
     of the LOC_xxx constants.
     """
+    if not cintf.IS_SUPPORTED:
+        return
     i = 0
     while i < array_length - 1:
         tag = array_p[i]
diff --git a/rpython/rlib/unicodedata/generate_unicodedb.py b/rpython/rlib/unicodedata/generate_unicodedb.py
--- a/rpython/rlib/unicodedata/generate_unicodedb.py
+++ b/rpython/rlib/unicodedata/generate_unicodedb.py
@@ -913,8 +913,17 @@
 
     casefolds = {}
     for code, char in table.enum_chars():
-        if char.casefolding and char.casefolding != [char.lower]:
-            casefolds[code] = char.casefolding
+        full_casefold = char.casefolding
+        if full_casefold is None:
+            full_casefold = [code]
+        full_lower = char.lower
+        if full_lower is None:
+            full_lower = code
+        # if we don't write anything into the file, then the RPython
+        # program would compute the result 'full_lower' instead.
+        # Is that the right answer?
+        if full_casefold != [full_lower]:
+            casefolds[code] = full_casefold
     writeDict(outfile, '_casefolds', casefolds, base_mod)
     print >> outfile, '''
 
diff --git a/rpython/rlib/unicodedata/test/test_unicodedata.py b/rpython/rlib/unicodedata/test/test_unicodedata.py
--- a/rpython/rlib/unicodedata/test/test_unicodedata.py
+++ b/rpython/rlib/unicodedata/test/test_unicodedata.py
@@ -148,3 +148,15 @@
     def test_changed_in_version_8(self):
         assert unicodedb_6_2_0.toupper_full(0x025C) == [0x025C]
         assert unicodedb_8_0_0.toupper_full(0x025C) == [0xA7AB]
+
+    def test_casefold(self):
+        # returns None when we have no special casefolding rule,
+        # which means that tolower_full() should be used instead
+        assert unicodedb_8_0_0.casefold_lookup(0x1000) == None
+        assert unicodedb_8_0_0.casefold_lookup(0x0061) == None
+        assert unicodedb_8_0_0.casefold_lookup(0x0041) == None
+        # a case where casefold() != lower()
+        assert unicodedb_8_0_0.casefold_lookup(0x00DF) == [ord('s'), ord('s')]
+        # returns the argument itself, and not None, in rare cases
+        # where tolower_full() would return something different
+        assert unicodedb_8_0_0.casefold_lookup(0x13A0) == [0x13A0]
diff --git a/rpython/rlib/unicodedata/unicodedb_8_0_0.py b/rpython/rlib/unicodedata/unicodedb_8_0_0.py
--- a/rpython/rlib/unicodedata/unicodedb_8_0_0.py
+++ b/rpython/rlib/unicodedata/unicodedb_8_0_0.py
@@ -21307,6 +21307,92 @@
         return code
 
 _casefolds = {
+5024: [5024],
+5025: [5025],
+5026: [5026],
+5027: [5027],
+5028: [5028],
+5029: [5029],
+5030: [5030],
+5031: [5031],
+5032: [5032],
+5033: [5033],
+5034: [5034],
+5035: [5035],
+5036: [5036],
+5037: [5037],
+5038: [5038],
+5039: [5039],
+5040: [5040],
+5041: [5041],
+5042: [5042],
+5043: [5043],
+5044: [5044],
+5045: [5045],
+5046: [5046],
+5047: [5047],
+5048: [5048],
+5049: [5049],
+5050: [5050],
+5051: [5051],
+5052: [5052],
+5053: [5053],
+5054: [5054],
+5055: [5055],
+5056: [5056],

From pypy.commits at gmail.com  Mon Jul 23 12:31:08 2018
From: pypy.commits at gmail.com (arigo)
Date: Mon, 23 Jul 2018 09:31:08 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Remove this get_raw_address() now that we
 have one in the base class
Message-ID: <5b5602cc.1c69fb81.6de6.880f@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r94886:8e5a7f8a9bf2
Date: 2018-07-23 18:30 +0200
http://bitbucket.org/pypy/pypy/changeset/8e5a7f8a9bf2/

Log:	Remove this get_raw_address() now that we have one in the base class

diff --git a/pypy/module/_io/interp_bytesio.py b/pypy/module/_io/interp_bytesio.py
--- a/pypy/module/_io/interp_bytesio.py
+++ b/pypy/module/_io/interp_bytesio.py
@@ -48,9 +48,6 @@
         finally:
             w_bytesio.seek(tell)
 
-    def get_raw_address(self):
-        raise ValueError("BytesIOBuffer does not have a raw address")
-
 
 class W_BytesIO(W_BufferedIOBase):
     import_from_mixin(RStringIO)

From pypy.commits at gmail.com  Thu Jul 26 12:38:17 2018
From: pypy.commits at gmail.com (wlav)
Date: Thu, 26 Jul 2018 09:38:17 -0700 (PDT)
Subject: [pypy-commit] pypy cppyy-packaging: fix incorrect return type
Message-ID: <5b59f8f9.1c69fb81.ac41.c333@mx.google.com>

Author: Wim Lavrijsen <WLavrijsen at lbl.gov>
Branch: cppyy-packaging
Changeset: r94887:76fe462b0a9e
Date: 2018-07-25 23:30 -0700
http://bitbucket.org/pypy/pypy/changeset/76fe462b0a9e/

Log:	fix incorrect return type

diff --git a/pypy/module/_cppyy/capi/loadable_capi.py b/pypy/module/_cppyy/capi/loadable_capi.py
--- a/pypy/module/_cppyy/capi/loadable_capi.py
+++ b/pypy/module/_cppyy/capi/loadable_capi.py
@@ -620,7 +620,7 @@
     return space.bool_w(call_capi(space, 'is_enum_data', args))
 def c_get_dimension_size(space, cppscope, datamember_index, dim_idx):
     args = [_ArgH(cppscope.handle), _ArgL(datamember_index), _ArgL(dim_idx)]
-    return space.bool_w(call_capi(space, 'get_dimension_size', args))
+    return space.int_w(call_capi(space, 'get_dimension_size', args))
 
 # misc helpers ---------------------------------------------------------------
 def c_strtoll(space, svalue):

From pypy.commits at gmail.com  Thu Jul 26 12:38:19 2018
From: pypy.commits at gmail.com (wlav)
Date: Thu, 26 Jul 2018 09:38:19 -0700 (PDT)
Subject: [pypy-commit] pypy cppyy-packaging: support for multi-dimensional
 arrays of instances
Message-ID: <5b59f8fb.1c69fb81.6faec.f0c4@mx.google.com>

Author: Wim Lavrijsen <WLavrijsen at lbl.gov>
Branch: cppyy-packaging
Changeset: r94888:9bc5b83ea8b9
Date: 2018-07-25 23:31 -0700
http://bitbucket.org/pypy/pypy/changeset/9bc5b83ea8b9/

Log:	support for multi-dimensional arrays of instances

diff --git a/pypy/module/_cppyy/converter.py b/pypy/module/_cppyy/converter.py
--- a/pypy/module/_cppyy/converter.py
+++ b/pypy/module/_cppyy/converter.py
@@ -625,16 +625,18 @@
 class InstanceArrayConverter(InstancePtrConverter):
     _immutable_fields_ = ['size']
 
-    def __init__(self, space, clsdecl, array_size):
+    def __init__(self, space, clsdecl, array_size, dimensions):
         InstancePtrConverter.__init__(self, space, clsdecl)
-        if array_size <= 0:
+        if array_size <= 0 or array_size == 2**31-1:   # cling's code for "unknown" (?)
             self.size = sys.maxint
         else:
             self.size = array_size
+        # peel one off as that should be the same as the array size
+        self.dimensions = dimensions[1:]
 
     def from_memory(self, space, w_obj, offset):
         address = rffi.cast(capi.C_OBJECT, self._get_raw_address(space, w_obj, offset))
-        return lowlevelviews.W_ArrayOfInstances(space, self.clsdecl, address, self.size)
+        return lowlevelviews.W_ArrayOfInstances(space, self.clsdecl, address, self.size, self.dimensions)
 
     def to_memory(self, space, w_obj, w_value, offset):
         self._is_abstract(space)
@@ -872,7 +874,16 @@
         pass
 
     # arrays (array_size may be negative, meaning: no size or no size found)
-    array_size = helper.array_size(_name)     # uses original arg
+    array_size = -1
+    if cpd == "[]":
+        array_size = helper.array_size(_name)    # uses original arg
+    elif cpd == '*' and ':' in default:
+        # this happens for multi-dimensional arrays: those are described as pointers
+        cpd = "[]"
+        splitpos = default.find(':')
+        if 0 < splitpos:     # always true, but needed for annotator
+            array_size = int(default[:splitpos])
+
     try:
         # TODO: using clean_name here drops const (e.g. const char[] will
         # never be seen this way)
@@ -908,8 +919,9 @@
         elif cpd in ["**", "*[]", "&*"]:
             return InstancePtrPtrConverter(space, clsdecl)
         elif cpd == "[]" and array_size > 0:
-            # TODO: retrieve dimensions
-            return InstanceArrayConverter(space, clsdecl, array_size)
+            # default encodes the dimensions
+            dims = default.split(':')
+            return InstanceArrayConverter(space, clsdecl, array_size, dims)
         elif cpd == "":
             return InstanceConverter(space, clsdecl)
     elif "(anonymous)" in name:

From pypy.commits at gmail.com  Thu Jul 26 12:38:21 2018
From: pypy.commits at gmail.com (wlav)
Date: Thu, 26 Jul 2018 09:38:21 -0700 (PDT)
Subject: [pypy-commit] pypy cppyy-packaging: support for multi-dimensional
 arrays of instances and further
Message-ID: <5b59f8fd.1c69fb81.2fc10.47ce@mx.google.com>

Author: Wim Lavrijsen <WLavrijsen at lbl.gov>
Branch: cppyy-packaging
Changeset: r94889:6f18d2274d2f
Date: 2018-07-25 23:32 -0700
http://bitbucket.org/pypy/pypy/changeset/6f18d2274d2f/

Log:	support for multi-dimensional arrays of instances and further
	refined template support

diff --git a/pypy/module/_cppyy/interp_cppyy.py b/pypy/module/_cppyy/interp_cppyy.py
--- a/pypy/module/_cppyy/interp_cppyy.py
+++ b/pypy/module/_cppyy/interp_cppyy.py
@@ -788,6 +788,7 @@
 class TemplateOverloadMixin(object):
     """Mixin to instantiate templated methods/functions."""
 
+    _attrs_ = ['tmpl_args_w']
     _mixin_ = True
 
     def construct_template_args(self, w_tpArgs, args_w = None):
@@ -840,23 +841,37 @@
         return cppol
 
     def instantiate_and_call(self, name, args_w):
-        # try to match with run-time instantiations
-        for cppol in self.master.overloads.values():
-            try:
-                if not self.space.is_w(self.w_this, self.space.w_None):
-                    return self.space.call_obj_args(cppol, self.w_this, Arguments(self.space, args_w))
-                return self.space.call_args(cppol, Arguments(self.space, args_w))
-            except Exception:
-                pass    # completely ignore for now; have to see whether errors become confusing
+        # existing cached instantiations
+        method = None
+        try:
+            if name[-1] == '>':   # full templated name, so ensure explicit
+                method = self.master.overloads[name]
+            else:
+            # try to match with run-time instantiations
+                # TODO: logically, this could be used, but in practice, it's proving to
+                #  greedy ... maybe as a last resort?
+                #for cppol in self.master.overloads.values():
+                #    try:
+                #        if not self.space.is_w(self.w_this, self.space.w_None):
+                #            return self.space.call_obj_args(cppol, self.w_this, Arguments(self.space, args_w))
+                #        return self.space.call_args(cppol, Arguments(self.space, args_w))
+                #    except Exception:
+                #        pass    # completely ignore for now; have to see whether errors become confusing
+                raise TypeError("pre-existing overloads failed")
+        except (KeyError, TypeError):
+            # if not known, try to deduce from argument types
+            w_types = self.space.newtuple([self.space.type(obj_w) for obj_w in args_w])
+            proto = self.construct_template_args(w_types, args_w)
+            method = self.find_method_template(name, proto)
 
-        # if all failed, then try to deduce from argument types
-        w_types = self.space.newtuple([self.space.type(obj_w) for obj_w in args_w])
-        proto = self.construct_template_args(w_types, args_w)
-        method = self.find_method_template(name, proto)
-
-        # only cache result if the name retains the full template
-        fullname = capi.c_method_full_name(self.space, method.functions[0].cppmethod)
-        if 0 <= fullname.rfind('>'):
+            # only cache result if the name retains the full template
+            # TODO: the problem is in part that c_method_full_name returns incorrect names,
+            # e.g. when default template arguments are involved, so for now use 'name' if it
+            # has the full templated name
+            if name[-1] == '>':
+                fullname = name
+            else:
+                fullname = capi.c_method_full_name(self.space, method.functions[0].cppmethod)
             try:
                 existing = self.master.overloads[fullname]
                 allf = existing.functions + method.functions
@@ -868,9 +883,10 @@
             except KeyError:
                 self.master.overloads[fullname] = method
 
-        if not self.space.is_w(self.w_this, self.space.w_None):
-            return self.space.call_obj_args(method, self.w_this, Arguments(self.space, args_w))
-        return self.space.call_args(method, Arguments(self.space, args_w))
+        if method is not None:
+            if not self.space.is_w(self.w_this, self.space.w_None):
+                return self.space.call_obj_args(method, self.w_this, Arguments(self.space, args_w))
+            return self.space.call_args(method, Arguments(self.space, args_w))
 
     def getitem_impl(self, name, args_w):
         space = self.space
@@ -884,14 +900,9 @@
         fullname = name+'<'+tmpl_args+'>'
         try:
             method = self.master.overloads[fullname]
-        except KeyError:
-            method = self.find_method_template(fullname)
-            # cache result (name is always full templated name)
-            self.master.overloads[fullname] = method
-            # also cache on "official" name (may include default template arguments)
-            c_fullname = capi.c_method_full_name(self.space, method.functions[0].cppmethod)
-            if c_fullname != fullname:
-                self.master.overloads[c_fullname] = method
+        except KeyError as e:
+            # defer instantiation until arguments are known
+            return self.clone(tmpl_args)
 
         return method.descr_get(self.w_this, None)
 
@@ -900,21 +911,29 @@
     """App-level dispatcher to allow both lookup/instantiation of templated methods and
     dispatch among overloads between templated and non-templated method."""
 
-    _attrs_ = ['name', 'overloads', 'master', 'w_this']
-    _immutable_fields_ = ['name']
+    _attrs_ = ['name', 'tmpl_args', 'overloads', 'master', 'w_this']
+    _immutable_fields_ = ['name', 'tmpl_args']
 
-    def __init__(self, space, name, decl_scope, functions, flags = OVERLOAD_FLAGS_USE_FFI):
+    def __init__(self, space, name, tmpl_args, decl_scope, functions, flags = OVERLOAD_FLAGS_USE_FFI):
          W_CPPOverload.__init__(self, space, decl_scope, functions, flags)
          self.name = name
+         self.tmpl_args = tmpl_args
          self.overloads = {}
          self.master = self
          self.w_this = space.w_None
 
+    def clone(self, tmpl_args):
+        other = W_CPPTemplateOverload(self.space, self.name, tmpl_args, self.scope, self.functions, self.flags)
+        other.overloads = self.overloads
+        other.master = self.master
+        other.w_this = self.w_this
+        return other
+
     def descr_get(self, w_cppinstance, w_cls=None):
         # TODO: don't return copy, but bind in an external object (like W_CPPOverload)
         if self.space.is_w(w_cppinstance, self.space.w_None):
             return self  # unbound, so no new instance needed
-        cppol = W_CPPTemplateOverload(self.space, self.name, self.scope, self.functions, self.flags)
+        cppol = W_CPPTemplateOverload(self.space, self.name, self.tmpl_args, self.scope, self.functions, self.flags)
         cppol.w_this = w_cppinstance
         cppol.master = self.master
         return cppol     # bound
@@ -924,13 +943,18 @@
         # direct call: either pick non-templated overload or attempt to deduce
         # the template instantiation from the argument types
 
-        # try existing overloads or compile-time instantiations
+        # do explicit lookup with tmpl_args if given
         try:
-            return W_CPPOverload.call_args(self, [self.w_this]+args_w)
+            fullname = self.name
+            if self.tmpl_args is not None:
+                fullname = fullname+'<'+self.tmpl_args+'>'
+            return self.instantiate_and_call(fullname, args_w)
         except Exception:
             pass
 
-        return self.instantiate_and_call(self.name, args_w)
+        # otherwise, try existing overloads or compile-time instantiations
+        # TODO: consolidate errors
+        return W_CPPOverload.call_args(self, [self.w_this]+args_w)
 
     @unwrap_spec(args_w='args_w')
     def getitem(self, args_w):
@@ -958,22 +982,30 @@
     """App-level dispatcher to allow both lookup/instantiation of templated methods and
     dispatch among overloads between templated and non-templated method."""
 
-    _attrs_ = ['name', 'overloads', 'master', 'w_this']
-    _immutable_fields_ = ['name']
+    _attrs_ = ['name', 'tmpl_args', 'overloads', 'master', 'w_this']
+    _immutable_fields_ = ['name', 'tmpl_args']
 
-    def __init__(self, space, name, decl_scope, funcs, flags = OVERLOAD_FLAGS_USE_FFI):
+    def __init__(self, space, name, tmpl_args, decl_scope, funcs, flags = OVERLOAD_FLAGS_USE_FFI):
          W_CPPStaticOverload.__init__(self, space, decl_scope, funcs, flags)
          self.name = name
+         self.tmpl_args = tmpl_args
          self.overloads = {}
          self.master = self
          self.w_this = space.w_None
 
+    def clone(self, tmpl_args):
+        other = W_CPPTemplateOverload(self.space, self.name, tmpl_args, self.scope, self.functions, self.flags)
+        other.overloads = self.overloads
+        other.master = self.master
+        other.w_this = self.w_this
+        return other
+
     def descr_get(self, w_cppinstance, w_cls=None):
         # TODO: don't return copy, but bind in an external object (like W_CPPOverload)
         if isinstance(w_cppinstance, W_CPPInstance):
             cppinstance = self.space.interp_w(W_CPPInstance, w_cppinstance)
             if cppinstance.clsdecl.handle != self.scope.handle:
-                cppol = W_CPPTemplateStaticOverload(self.space, self.name, self.scope, self.functions, self.flags)
+                cppol = W_CPPTemplateStaticOverload(self.space, self.name, self.tmpl_args, self.scope, self.functions, self.flags)
                 cppol.w_this = w_cppinstance
                 cppol.master = self.master
                 return cppol       # bound
@@ -983,15 +1015,20 @@
     def call_args(self, args_w):
         # direct call: either pick non-templated overload or attempt to deduce
         # the template instantiation from the argument types
+        # TODO: refactor with W_CPPTemplateOverload
 
-        # try existing overloads or compile-time instantiations
+        # do explicit lookup with tmpl_args if given
         try:
-            return W_CPPStaticOverload.call_args(self, args_w)
+            fullname = self.name
+            if self.tmpl_args is not None:
+                fullname = fullname+'<'+self.tmpl_args+'>'
+            return self.instantiate_and_call(fullname, args_w)
         except Exception:
             pass
 
-        # try new instantiation
-        return self.instantiate_and_call(self.name, args_w)
+        # otherwise, try existing overloads or compile-time instantiations
+        # TODO: consolidate errors
+        return W_CPPStaticOverload.call_args(self, args_w)
 
     @unwrap_spec(args_w='args_w')
     def getitem(self, args_w):
@@ -1036,10 +1073,10 @@
     _attrs_ = ['space', 'scope', 'converter', 'offset']
     _immutable_fields = ['scope', 'converter', 'offset']
 
-    def __init__(self, space, decl_scope, type_name, offset):
+    def __init__(self, space, decl_scope, type_name, dimensions, offset):
         self.space = space
         self.scope = decl_scope
-        self.converter = converter.get_converter(self.space, type_name, '')
+        self.converter = converter.get_converter(self.space, type_name, dimensions)
         self.offset = rffi.cast(rffi.LONG, offset)
 
     def _get_offset(self, cppinstance):
@@ -1184,6 +1221,15 @@
         self.datamembers[name] = new_dm
         return new_dm
 
+    def _encode_dm_dimensions(self, idata):
+        # encode dimensions (TODO: this is ugly, but here's where the info is)
+        dims = []
+        sz = capi.c_get_dimension_size(self.space, self, idata, len(dims))
+        while 0 < sz:
+            dims.append(str(sz))
+            sz = capi.c_get_dimension_size(self.space, self, idata, len(dims))
+        return ':'.join(dims)
+
     @unwrap_spec(name='text', signature='text')
     def scope__dispatch__(self, name, signature):
         overload = self.get_overload(name)
@@ -1226,10 +1272,11 @@
         offset = capi.c_datamember_offset(self.space, self, dm_idx)
         if offset == -1:
             raise self.missing_attribute_error(dm_name)
+        dims = self._encode_dm_dimensions(dm_idx)
         if capi.c_is_const_data(self.space, self, dm_idx):
-            datamember = W_CPPConstStaticData(self.space, self, type_name, offset)
+            datamember = W_CPPConstStaticData(self.space, self, type_name, dims, offset)
         else:
-            datamember = W_CPPStaticData(self.space, self, type_name, offset)
+            datamember = W_CPPStaticData(self.space, self, type_name, dims, offset)
         self.datamembers[dm_name] = datamember
         return datamember
 
@@ -1244,10 +1291,10 @@
                 if capi.c_method_is_template(self.space, self, idx):
                     templated = True
             if templated:
-                return W_CPPTemplateStaticOverload(self.space, meth_name, self, cppfunctions[:])
+                return W_CPPTemplateStaticOverload(self.space, meth_name, None, self, cppfunctions[:])
             return W_CPPStaticOverload(self.space, self, cppfunctions[:])
         elif capi.c_exists_method_template(self.space, self, meth_name):
-            return W_CPPTemplateStaticOverload(self.space, meth_name, self, [])
+            return W_CPPTemplateStaticOverload(self.space, meth_name, None, self, [])
         raise self.missing_attribute_error(meth_name)
 
     def find_datamember(self, dm_name):
@@ -1339,12 +1386,12 @@
             elif ftype & FUNCTION_IS_STATIC:
                 if ftype & FUNCTION_IS_TEMPLATE:
                     cppname = capi.c_method_name(self.space, methods[0].cppmethod)
-                    overload = W_CPPTemplateStaticOverload(self.space, cppname, self, methods[:])
+                    overload = W_CPPTemplateStaticOverload(self.space, cppname, None, self, methods[:])
                 else:
                     overload = W_CPPStaticOverload(self.space, self, methods[:])
             elif ftype & FUNCTION_IS_TEMPLATE:
                 cppname = capi.c_method_name(self.space, methods[0].cppmethod)
-                overload = W_CPPTemplateOverload(self.space, cppname, self, methods[:])
+                overload = W_CPPTemplateOverload(self.space, cppname, None, self, methods[:])
             else:
                 overload = W_CPPOverload(self.space, self, methods[:])
             self.overloads[pyname] = overload
@@ -1384,14 +1431,15 @@
                 continue      # dictionary problem; raises AttributeError on use
             is_static = bool(capi.c_is_staticdata(self.space, self, i))
             is_const  = bool(capi.c_is_const_data(self.space, self, i))
+            dims = self._encode_dm_dimensions(i)
             if is_static and is_const:
-                datamember = W_CPPConstStaticData(self.space, self, type_name, offset)
+                datamember = W_CPPConstStaticData(self.space, self, type_name, dims, offset)
             elif is_static:
-                datamember = W_CPPStaticData(self.space, self, type_name, offset)
+                datamember = W_CPPStaticData(self.space, self, type_name, dims, offset)
             elif is_const:
-                datamember = W_CPPConstDataMember(self.space, self, type_name, offset)
+                datamember = W_CPPConstDataMember(self.space, self, type_name, dims, offset)
             else:
-                datamember = W_CPPDataMember(self.space, self, type_name, offset)
+                datamember = W_CPPDataMember(self.space, self, type_name, dims, offset)
             self.datamembers[datamember_name] = datamember
 
     def find_overload(self, meth_name):

From pypy.commits at gmail.com  Thu Jul 26 12:38:24 2018
From: pypy.commits at gmail.com (wlav)
Date: Thu, 26 Jul 2018 09:38:24 -0700 (PDT)
Subject: [pypy-commit] pypy cppyy-packaging: multi-dimensional arrays of
 instances
Message-ID: <5b59f900.1c69fb81.ff1b0.00f6@mx.google.com>

Author: Wim Lavrijsen <WLavrijsen at lbl.gov>
Branch: cppyy-packaging
Changeset: r94890:a5e401a889f6
Date: 2018-07-25 23:32 -0700
http://bitbucket.org/pypy/pypy/changeset/a5e401a889f6/

Log:	multi-dimensional arrays of instances

diff --git a/pypy/module/_cppyy/lowlevelviews.py b/pypy/module/_cppyy/lowlevelviews.py
--- a/pypy/module/_cppyy/lowlevelviews.py
+++ b/pypy/module/_cppyy/lowlevelviews.py
@@ -9,6 +9,7 @@
 from pypy.interpreter.baseobjspace import W_Root
 
 from rpython.rtyper.lltypesystem import rffi
+from rpython.rlib.rarithmetic import intmask
 
 from pypy.module._rawffi.array import W_ArrayInstance
 from pypy.module._rawffi.interp_rawffi import segfault_exception
@@ -63,11 +64,17 @@
     _attrs_ = ['converter', 'baseaddress', 'clssize', 'length']
     _immutable_fields_ = ['converter', 'baseaddress', 'clssize']
 
-    def __init__(self, space, clsdecl, address, length):
+    def __init__(self, space, clsdecl, address, length, dimensions):
         from pypy.module._cppyy import converter
-        self.converter   = converter.get_converter(space, clsdecl.name, '')
+        name = clsdecl.name
+        self.clssize = int(intmask(capi.c_size_of_klass(space, clsdecl)))
+        if dimensions:
+            name = name + '[' + dimensions[0] + ']'
+            for num in dimensions:
+                self.clssize *= int(num)
+        dimensions = ':'.join(dimensions)
+        self.converter   = converter.get_converter(space, name, dimensions)
         self.baseaddress = address
-        self.clssize     = capi.c_size_of_klass(space, clsdecl)
         self.length      = length
 
     @unwrap_spec(idx=int)
@@ -76,15 +83,19 @@
             raise segfault_exception(space, "accessing elements of freed array")
         if idx >= self.length or idx < 0:
             raise OperationError(space.w_IndexError, space.w_None)
-        itemaddress = rffi.cast(rffi.LONG, self.baseaddress+idx*self.clssize)
+        itemaddress = rffi.cast(rffi.LONG, self.baseaddress)+idx*self.clssize
         return self.converter.from_memory(space, space.w_None, itemaddress)
 
     def getlength(self, space):
         return space.newint(self.length)
 
+    def setlength(self, space, w_length):
+        self.length = space.int_w(w_length)
+
 W_ArrayOfInstances.typedef = TypeDef(
     'ArrayOfInstances',
     __getitem__ = interp2app(W_ArrayOfInstances.getitem),
     __len__     = interp2app(W_ArrayOfInstances.getlength),
+    size        = GetSetProperty(W_ArrayOfInstances.getlength, W_ArrayOfInstances.setlength),
 )
 W_ArrayOfInstances.typedef.acceptable_as_base_class = False

From pypy.commits at gmail.com  Thu Jul 26 12:38:26 2018
From: pypy.commits at gmail.com (wlav)
Date: Thu, 26 Jul 2018 09:38:26 -0700 (PDT)
Subject: [pypy-commit] pypy cppyy-packaging: more template tests
Message-ID: <5b59f902.1c69fb81.cc726.c0f9@mx.google.com>

Author: Wim Lavrijsen <WLavrijsen at lbl.gov>
Branch: cppyy-packaging
Changeset: r94891:57b26d4f25d5
Date: 2018-07-25 23:32 -0700
http://bitbucket.org/pypy/pypy/changeset/57b26d4f25d5/

Log:	more template tests

diff --git a/pypy/module/_cppyy/test/templates.h b/pypy/module/_cppyy/test/templates.h
--- a/pypy/module/_cppyy/test/templates.h
+++ b/pypy/module/_cppyy/test/templates.h
@@ -1,5 +1,6 @@
 #include <string>
 #include <sstream>
+#include <vector>
 
 
 //===========================================================================
@@ -56,6 +57,18 @@
     return 13;
 }
 
+template <typename F>
+struct SomeResult {
+    F m_retval;
+};
+
+template <class I, typename O = float>
+SomeResult<O> global_get_some_result(const std::vector<I>& carrier) {
+    SomeResult<O> r{};
+    r.m_retval = O(carrier[0]);
+    return r;
+}
+
 
 //===========================================================================
 // variadic functions
diff --git a/pypy/module/_cppyy/test/test_datatypes.py b/pypy/module/_cppyy/test/test_datatypes.py
--- a/pypy/module/_cppyy/test/test_datatypes.py
+++ b/pypy/module/_cppyy/test/test_datatypes.py
@@ -61,6 +61,30 @@
         #assert round(c.get_ldouble_r()  + 88., 24) == 0
         assert round(c.m_double + 77., 8) == 0
 
+        # complex<double> type
+        assert type(c.get_complex()) == complex
+        assert round(c.get_complex().real    -  99., 11) == 0
+        assert round(c.get_complex().imag    - 101., 11) == 0
+        assert repr(c.get_complex()) == '(99+101j)'
+        assert round(c.get_complex_cr().real -  99., 11) == 0
+        assert round(c.get_complex_cr().imag - 101., 11) == 0
+        assert round(c.get_complex_r().real  -  99., 11) == 0
+        assert round(c.get_complex_r().imag  - 101., 11) == 0
+        assert complex(cppyy.gbl.std.complex['double'](1, 2)) == complex(1, 2)
+
+        # complex<int> retains C++ type in all cases (but includes pythonization to
+        # resemble Python's complex more closely
+        assert type(c.get_icomplex()) == cppyy.gbl.std.complex[int]
+        assert round(c.get_icomplex().real    - 121., 11) == 0
+        assert round(c.get_icomplex().imag    - 141., 11) == 0
+        assert repr(c.get_icomplex()) == '(121+141j)'
+        assert round(c.get_icomplex_cr().real - 121., 11) == 0
+        assert round(c.get_icomplex_cr().imag - 141., 11) == 0
+        assert type(c.get_icomplex_r()) == cppyy.gbl.std.complex[int]
+        assert round(c.get_icomplex_r().real  - 121., 11) == 0
+        assert round(c.get_icomplex_r().imag  - 141., 11) == 0
+        assert complex(cppyy.gbl.std.complex['int'](1, 2)) == complex(1, 2)
+
         # reading of enum types
         assert c.m_enum == CppyyTestData.kNothing
         assert c.m_enum == c.kNothing
diff --git a/pypy/module/_cppyy/test/test_stltypes.py b/pypy/module/_cppyy/test/test_stltypes.py
--- a/pypy/module/_cppyy/test/test_stltypes.py
+++ b/pypy/module/_cppyy/test/test_stltypes.py
@@ -210,8 +210,6 @@
         vb[-1] = True
         assert vb[7]
 
-        print [x for x in vb]
-
         assert [x for x in vb] == [True]+[False]*6+[True]
 
         assert len(vb[4:8]) == 4
@@ -224,7 +222,8 @@
     def setup_class(cls):
         cls.w_test_dct  = cls.space.newtext(test_dct)
         cls.w_stlstring = cls.space.appexec([], """():
-            import ctypes
+            import ctypes, _cppyy
+            _cppyy._post_import_startup()
             return ctypes.CDLL(%r, ctypes.RTLD_GLOBAL)""" % (test_dct, ))
 
     def test01_string_argument_passing(self):
@@ -295,6 +294,33 @@
         assert t0 == c.get_string1()
         assert s == c.get_string1()
 
+    def test04_array_of_strings(self):
+        """Access to global arrays of strings"""
+
+        import _cppyy as cppyy
+
+        assert tuple(cppyy.gbl.str_array_1) == ('a', 'b', 'c')
+        str_array_2 = cppyy.gbl.str_array_2
+        # fix up the size
+        str_array_2.size = 4
+        assert tuple(str_array_2) == ('d', 'e', 'f', 'g')
+        assert tuple(str_array_2) == ('d', 'e', 'f', 'g')
+
+        # multi-dimensional
+        vals = ['a', 'b', 'c', 'd', 'e', 'f']
+        str_array_3 = cppyy.gbl.str_array_3
+        for i in range(3):
+            for j in range(2):
+                assert str_array_3[i][j] == vals[i*2+j]
+
+        vals = ['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h',
+                'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p']
+        str_array_4 = cppyy.gbl.str_array_4
+        for i in range(4):
+            for j in range(2):
+                for k in range(2):
+                    assert str_array_4[i][j][k] == vals[i*4+j*2+k]
+
 
 class AppTestSTLLIST:
     spaceconfig = dict(usemodules=['_cppyy', '_rawffi', 'itertools'])
@@ -303,7 +329,8 @@
         cls.w_N = cls.space.newint(13)
         cls.w_test_dct  = cls.space.newtext(test_dct)
         cls.w_stlstring = cls.space.appexec([], """():
-            import ctypes
+            import ctypes, _cppyy
+            _cppyy._post_import_startup()
             return ctypes.CDLL(%r, ctypes.RTLD_GLOBAL)""" % (test_dct, ))
 
     def test01_builtin_list_type(self):
@@ -359,7 +386,8 @@
         cls.w_N = cls.space.newint(13)
         cls.w_test_dct  = cls.space.newtext(test_dct)
         cls.w_stlstring = cls.space.appexec([], """():
-            import ctypes
+            import ctypes, _cppyy
+            _cppyy._post_import_startup()
             return ctypes.CDLL(%r, ctypes.RTLD_GLOBAL)""" % (test_dct, ))
 
     def test01_builtin_map_type(self):
@@ -468,7 +496,8 @@
     def setup_class(cls):
         cls.w_test_dct  = cls.space.newtext(test_dct)
         cls.w_stlstring = cls.space.appexec([], """():
-            import ctypes
+            import ctypes, _cppyy
+            _cppyy._post_import_startup()
             return ctypes.CDLL(%r, ctypes.RTLD_GLOBAL)""" % (test_dct, ))
 
     def test01_builtin_vector_iterators(self):
@@ -505,7 +534,8 @@
     def setup_class(cls):
         cls.w_test_dct  = cls.space.newtext(test_dct)
         cls.w_stlstring = cls.space.appexec([], """():
-            import ctypes
+            import ctypes, _cppyy
+            _cppyy._post_import_startup()
             return ctypes.CDLL(%r, ctypes.RTLD_GLOBAL)""" % (test_dct, ))
 
     def test01_explicit_templates(self):
diff --git a/pypy/module/_cppyy/test/test_templates.py b/pypy/module/_cppyy/test/test_templates.py
--- a/pypy/module/_cppyy/test/test_templates.py
+++ b/pypy/module/_cppyy/test/test_templates.py
@@ -53,32 +53,53 @@
     def test03_templated_function(self):
         """Templated global and static functions lookup and calls"""
 
-        import _cppyy
+        import _cppyy as cppyy
 
         # TODO: the following only works if something else has already
         # loaded the headers associated with this template
-        ggs = _cppyy.gbl.global_get_size
+        ggs = cppyy.gbl.global_get_size
         assert ggs['char']() == 1
 
-        gsf = _cppyy.gbl.global_some_foo
+        gsf = cppyy.gbl.global_some_foo
 
         assert gsf[int](3) == 42
         assert gsf(3)      == 42
         assert gsf(3.)     == 42
 
-        gsb = _cppyy.gbl.global_some_bar
+        gsb = cppyy.gbl.global_some_bar
 
         assert gsb(3)            == 13
         assert gsb['double'](3.) == 13
 
         # TODO: the following only works in a namespace
-        nsgsb = _cppyy.gbl.SomeNS.some_bar
+        nsgsb = cppyy.gbl.SomeNS.some_bar
 
         assert nsgsb[3]
         assert nsgsb[3]() == 3
 
         # TODO: add some static template method
 
+        # test forced creation of subsequent overloads
+        vector = cppyy.gbl.std.vector
+        # float in, float out
+        ggsr = cppyy.gbl.global_get_some_result['std::vector<float>']
+        assert type(ggsr(vector['float']([0.5])).m_retval) == float
+        assert ggsr(vector['float']([0.5])).m_retval == 0.5
+        # int in, float out
+        ggsr = cppyy.gbl.global_get_some_result['std::vector<int>']
+        assert type(ggsr(vector['int']([5])).m_retval) == float
+        assert ggsr(vector['int']([5])).m_retval == 5.
+        # float in, int out
+        # TODO: this now matches the earlier overload
+        #ggsr = cppyy.gbl.global_get_some_result['std::vector<float>, int']
+        #assert type(ggsr(vector['float']([0.3])).m_retval) == int
+        #assert ggsr(vector['float']([0.3])).m_retval == 0
+        # int in, int out
+        # TODO: same as above, matches earlier overload
+        #ggsr = cppyy.gbl.global_get_some_result['std::vector<int>, int']
+        #assert type(ggsr(vector['int']([5])).m_retval) == int
+        #assert ggsr(vector['int']([5])).m_retval == 5
+
     def test04_variadic_function(self):
         """Call a variadic function"""
 

From pypy.commits at gmail.com  Thu Jul 26 12:38:28 2018
From: pypy.commits at gmail.com (wlav)
Date: Thu, 26 Jul 2018 09:38:28 -0700 (PDT)
Subject: [pypy-commit] pypy cppyy-packaging: annotator fixes
Message-ID: <5b59f904.1c69fb81.b2d64.1877@mx.google.com>

Author: Wim Lavrijsen <WLavrijsen at lbl.gov>
Branch: cppyy-packaging
Changeset: r94892:92f43bfdbe30
Date: 2018-07-26 09:17 -0700
http://bitbucket.org/pypy/pypy/changeset/92f43bfdbe30/

Log:	annotator fixes

diff --git a/pypy/module/_cppyy/interp_cppyy.py b/pypy/module/_cppyy/interp_cppyy.py
--- a/pypy/module/_cppyy/interp_cppyy.py
+++ b/pypy/module/_cppyy/interp_cppyy.py
@@ -841,10 +841,10 @@
         return cppol
 
     def instantiate_and_call(self, name, args_w):
-        # existing cached instantiations
         method = None
         try:
-            if name[-1] == '>':   # full templated name, so ensure explicit
+            # existing cached instantiations
+            if name[-1] == '>':   # only accept full templated name, to ensure explicit
                 method = self.master.overloads[name]
             else:
             # try to match with run-time instantiations
@@ -994,7 +994,7 @@
          self.w_this = space.w_None
 
     def clone(self, tmpl_args):
-        other = W_CPPTemplateOverload(self.space, self.name, tmpl_args, self.scope, self.functions, self.flags)
+        other = W_CPPTemplateStaticOverload(self.space, self.name, tmpl_args, self.scope, self.functions, self.flags)
         other.overloads = self.overloads
         other.master = self.master
         other.w_this = self.w_this

From pypy.commits at gmail.com  Thu Jul 26 12:38:30 2018
From: pypy.commits at gmail.com (wlav)
Date: Thu, 26 Jul 2018 09:38:30 -0700 (PDT)
Subject: [pypy-commit] pypy cppyy-packaging: defer pythonization of
 std::complex for now
Message-ID: <5b59f906.1c69fb81.61100.2e96@mx.google.com>

Author: Wim Lavrijsen <WLavrijsen at lbl.gov>
Branch: cppyy-packaging
Changeset: r94893:95e2a7e64991
Date: 2018-07-26 09:17 -0700
http://bitbucket.org/pypy/pypy/changeset/95e2a7e64991/

Log:	defer pythonization of std::complex for now

diff --git a/pypy/module/_cppyy/test/test_datatypes.py b/pypy/module/_cppyy/test/test_datatypes.py
--- a/pypy/module/_cppyy/test/test_datatypes.py
+++ b/pypy/module/_cppyy/test/test_datatypes.py
@@ -61,7 +61,7 @@
         #assert round(c.get_ldouble_r()  + 88., 24) == 0
         assert round(c.m_double + 77., 8) == 0
 
-        # complex<double> type
+        """# complex<double> type
         assert type(c.get_complex()) == complex
         assert round(c.get_complex().real    -  99., 11) == 0
         assert round(c.get_complex().imag    - 101., 11) == 0
@@ -83,7 +83,7 @@
         assert type(c.get_icomplex_r()) == cppyy.gbl.std.complex[int]
         assert round(c.get_icomplex_r().real  - 121., 11) == 0
         assert round(c.get_icomplex_r().imag  - 141., 11) == 0
-        assert complex(cppyy.gbl.std.complex['int'](1, 2)) == complex(1, 2)
+        assert complex(cppyy.gbl.std.complex['int'](1, 2)) == complex(1, 2)"""
 
         # reading of enum types
         assert c.m_enum == CppyyTestData.kNothing

From pypy.commits at gmail.com  Fri Jul 27 16:02:14 2018
From: pypy.commits at gmail.com (mattip)
Date: Fri, 27 Jul 2018 13:02:14 -0700 (PDT)
Subject: [pypy-commit] pypy unicode-utf8: merge default into branch
Message-ID: <5b5b7a46.1c69fb81.6a212.5e81@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: unicode-utf8
Changeset: r94900:e32ab94a29ee
Date: 2018-07-11 09:25 -0500
http://bitbucket.org/pypy/pypy/changeset/e32ab94a29ee/

Log:	merge default into branch

diff too long, truncating to 2000 out of 2065 lines

diff --git a/lib-python/2.7/code.py b/lib-python/2.7/code.py
--- a/lib-python/2.7/code.py
+++ b/lib-python/2.7/code.py
@@ -104,6 +104,12 @@
         except SystemExit:
             raise
         except:
+            if softspace(sys.stdout, 0):
+                print
+            try:
+                sys.stdout.flush()
+            except:
+                pass
             self.showtraceback()
         else:
             if softspace(sys.stdout, 0):
diff --git a/lib-python/2.7/fractions.py b/lib-python/2.7/fractions.py
--- a/lib-python/2.7/fractions.py
+++ b/lib-python/2.7/fractions.py
@@ -517,8 +517,13 @@
             # Get integers right.
             return hash(self._numerator)
         # Expensive check, but definitely correct.
-        if self == float(self):
-            return hash(float(self))
+        # PyPy: the following 4 lines used to be almost twice slower:
+        #         if self == float(self):
+        #             return hash(float(self))
+        f = float(self)
+        x, y = f.as_integer_ratio()    # signs are correct: y is positive
+        if self._numerator == x and self._denominator == y:
+            return hash(f)
         else:
             # Use tuple's hash to avoid a high collision rate on
             # simple fractions.
diff --git a/lib_pypy/cffi.egg-info/PKG-INFO b/lib_pypy/cffi.egg-info/PKG-INFO
--- a/lib_pypy/cffi.egg-info/PKG-INFO
+++ b/lib_pypy/cffi.egg-info/PKG-INFO
@@ -1,6 +1,6 @@
 Metadata-Version: 1.1
 Name: cffi
-Version: 1.11.5
+Version: 1.12.0
 Summary: Foreign Function Interface for Python calling C code.
 Home-page: http://cffi.readthedocs.org
 Author: Armin Rigo, Maciej Fijalkowski
diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py
--- a/lib_pypy/cffi/__init__.py
+++ b/lib_pypy/cffi/__init__.py
@@ -4,8 +4,8 @@
 from .api import FFI
 from .error import CDefError, FFIError, VerificationError, VerificationMissing
 
-__version__ = "1.11.5"
-__version_info__ = (1, 11, 5)
+__version__ = "1.12.0"
+__version_info__ = (1, 12, 0)
 
 # The verifier module file names are based on the CRC32 of a string that
 # contains the following version number.  It may be older than __version__
diff --git a/lib_pypy/cffi/_embedding.h b/lib_pypy/cffi/_embedding.h
--- a/lib_pypy/cffi/_embedding.h
+++ b/lib_pypy/cffi/_embedding.h
@@ -221,7 +221,7 @@
 
         if (f != NULL && f != Py_None) {
             PyFile_WriteString("\nFrom: " _CFFI_MODULE_NAME
-                               "\ncompiled with cffi version: 1.11.5"
+                               "\ncompiled with cffi version: 1.12.0"
                                "\n_cffi_backend module: ", f);
             modules = PyImport_GetModuleDict();
             mod = PyDict_GetItemString(modules, "_cffi_backend");
diff --git a/lib_pypy/cffi/api.py b/lib_pypy/cffi/api.py
--- a/lib_pypy/cffi/api.py
+++ b/lib_pypy/cffi/api.py
@@ -96,18 +96,21 @@
             self.CData, self.CType = backend._get_types()
         self.buffer = backend.buffer
 
-    def cdef(self, csource, override=False, packed=False):
+    def cdef(self, csource, override=False, packed=False, pack=None):
         """Parse the given C source.  This registers all declared functions,
         types, and global variables.  The functions and global variables can
         then be accessed via either 'ffi.dlopen()' or 'ffi.verify()'.
         The types can be used in 'ffi.new()' and other functions.
         If 'packed' is specified as True, all structs declared inside this
         cdef are packed, i.e. laid out without any field alignment at all.
+        Alternatively, 'pack' can be a small integer, and requests for
+        alignment greater than that are ignored (pack=1 is equivalent to
+        packed=True).
         """
-        self._cdef(csource, override=override, packed=packed)
+        self._cdef(csource, override=override, packed=packed, pack=pack)
 
-    def embedding_api(self, csource, packed=False):
-        self._cdef(csource, packed=packed, dllexport=True)
+    def embedding_api(self, csource, packed=False, pack=None):
+        self._cdef(csource, packed=packed, pack=pack, dllexport=True)
         if self._embedding is None:
             self._embedding = ''
 
diff --git a/lib_pypy/cffi/backend_ctypes.py b/lib_pypy/cffi/backend_ctypes.py
--- a/lib_pypy/cffi/backend_ctypes.py
+++ b/lib_pypy/cffi/backend_ctypes.py
@@ -730,7 +730,8 @@
         return self._new_struct_or_union('union', name, ctypes.Union)
 
     def complete_struct_or_union(self, CTypesStructOrUnion, fields, tp,
-                                 totalsize=-1, totalalignment=-1, sflags=0):
+                                 totalsize=-1, totalalignment=-1, sflags=0,
+                                 pack=0):
         if totalsize >= 0 or totalalignment >= 0:
             raise NotImplementedError("the ctypes backend of CFFI does not support "
                                       "structures completed by verify(); please "
@@ -751,6 +752,8 @@
                 bfield_types[fname] = Ellipsis
         if sflags & 8:
             struct_or_union._pack_ = 1
+        elif pack:
+            struct_or_union._pack_ = pack
         struct_or_union._fields_ = cfields
         CTypesStructOrUnion._bfield_types = bfield_types
         #
diff --git a/lib_pypy/cffi/cparser.py b/lib_pypy/cffi/cparser.py
--- a/lib_pypy/cffi/cparser.py
+++ b/lib_pypy/cffi/cparser.py
@@ -306,11 +306,25 @@
             msg = 'parse error\n%s' % (msg,)
         raise CDefError(msg)
 
-    def parse(self, csource, override=False, packed=False, dllexport=False):
+    def parse(self, csource, override=False, packed=False, pack=None,
+                    dllexport=False):
+        if packed:
+            if packed != True:
+                raise ValueError("'packed' should be False or True; use "
+                                 "'pack' to give another value")
+            if pack:
+                raise ValueError("cannot give both 'pack' and 'packed'")
+            pack = 1
+        elif pack:
+            if pack & (pack - 1):
+                raise ValueError("'pack' must be a power of two, not %r" %
+                    (pack,))
+        else:
+            pack = 0
         prev_options = self._options
         try:
             self._options = {'override': override,
-                             'packed': packed,
+                             'packed': pack,
                              'dllexport': dllexport}
             self._internal_parse(csource)
         finally:
diff --git a/lib_pypy/cffi/model.py b/lib_pypy/cffi/model.py
--- a/lib_pypy/cffi/model.py
+++ b/lib_pypy/cffi/model.py
@@ -342,7 +342,7 @@
     fixedlayout = None
     completed = 0
     partial = False
-    packed = False
+    packed = 0
 
     def __init__(self, name, fldnames, fldtypes, fldbitsize, fldquals=None):
         self.name = name
@@ -414,11 +414,14 @@
             fldtypes = [tp.get_cached_btype(ffi, finishlist)
                         for tp in self.fldtypes]
             lst = list(zip(self.fldnames, fldtypes, self.fldbitsize))
-            sflags = 0
+            extra_flags = ()
             if self.packed:
-                sflags = 8    # SF_PACKED
+                if self.packed == 1:
+                    extra_flags = (8,)    # SF_PACKED
+                else:
+                    extra_flags = (0, self.packed)
             ffi._backend.complete_struct_or_union(BType, lst, self,
-                                                  -1, -1, sflags)
+                                                  -1, -1, *extra_flags)
             #
         else:
             fldtypes = []
diff --git a/lib_pypy/cffi/recompiler.py b/lib_pypy/cffi/recompiler.py
--- a/lib_pypy/cffi/recompiler.py
+++ b/lib_pypy/cffi/recompiler.py
@@ -893,6 +893,12 @@
             else:
                 flags.append("_CFFI_F_CHECK_FIELDS")
             if tp.packed:
+                if tp.packed > 1:
+                    raise NotImplementedError(
+                        "%r is declared with 'pack=%r'; only 0 or 1 are "
+                        "supported in API mode (try to use \"...;\", which "
+                        "does not require a 'pack' declaration)" %
+                        (tp, tp.packed))
                 flags.append("_CFFI_F_PACKED")
         else:
             flags.append("_CFFI_F_EXTERNAL")
diff --git a/pypy/doc/config/objspace.disable_entrypoints.txt b/pypy/doc/config/objspace.disable_entrypoints.txt
new file mode 100644
diff --git a/pypy/doc/config/objspace.fstrings.txt b/pypy/doc/config/objspace.fstrings.txt
new file mode 100644
diff --git a/pypy/doc/config/objspace.hash.txt b/pypy/doc/config/objspace.hash.txt
new file mode 100644
diff --git a/pypy/doc/config/objspace.usemodules._frozen_importlib.txt b/pypy/doc/config/objspace.usemodules._frozen_importlib.txt
new file mode 100644
diff --git a/pypy/doc/config/objspace.usemodules._jitlog.txt b/pypy/doc/config/objspace.usemodules._jitlog.txt
new file mode 100644
diff --git a/pypy/doc/config/objspace.usemodules.faulthandler.txt b/pypy/doc/config/objspace.usemodules.faulthandler.txt
new file mode 100644
diff --git a/pypy/doc/config/translation.backendopt.replace_we_are_jitted.txt b/pypy/doc/config/translation.backendopt.replace_we_are_jitted.txt
new file mode 100644
diff --git a/pypy/doc/config/translation.jit_opencoder_model.txt b/pypy/doc/config/translation.jit_opencoder_model.txt
new file mode 100644
diff --git a/pypy/doc/config/translation.keepgoing.txt b/pypy/doc/config/translation.keepgoing.txt
new file mode 100644
diff --git a/pypy/doc/config/translation.libname.txt b/pypy/doc/config/translation.libname.txt
new file mode 100644
diff --git a/pypy/doc/config/translation.lto.txt b/pypy/doc/config/translation.lto.txt
new file mode 100644
diff --git a/pypy/doc/config/translation.profoptargs.txt b/pypy/doc/config/translation.profoptargs.txt
new file mode 100644
diff --git a/pypy/doc/config/translation.reverse_debugger.txt b/pypy/doc/config/translation.reverse_debugger.txt
new file mode 100644
diff --git a/pypy/doc/config/translation.split_gc_address_space.txt b/pypy/doc/config/translation.split_gc_address_space.txt
new file mode 100644
diff --git a/pypy/doc/tool/makecontributor.py b/pypy/doc/tool/makecontributor.py
--- a/pypy/doc/tool/makecontributor.py
+++ b/pypy/doc/tool/makecontributor.py
@@ -18,12 +18,13 @@
     'Antonio Cuni': ['antocuni', 'anto'],
     'Armin Rigo': ['arigo', 'arfigo', 'armin', 'arigato'],
     'Maciej Fijalkowski': ['fijal'],
-    'Carl Friedrich Bolz-Tereick': ['Carl Friedrich Bolz', 'cfbolz', 'cf'],
+    'Carl Friedrich Bolz-Tereick': ['Carl Friedrich Bolz', 'cfbolz', 'cf', 'cbolz'],
     'Samuele Pedroni': ['pedronis', 'samuele', 'samule'],
-    'Richard Plangger':['planrich'],
-    'Michael Hudson': ['mwh'],
+    'Richard Plangger': ['planrich', 'plan_rich'],
+    'Remi Meier': ['remi'],
+    'Michael Hudson-Doyle': ['mwh', 'Michael Hudson'],
     'Holger Krekel': ['hpk', 'holger krekel', 'holger', 'hufpk'],
-    "Amaury Forgeot d'Arc": ['afa'],
+    "Amaury Forgeot d'Arc": ['afa', 'amauryfa at gmail.com'],
     'Alex Gaynor': ['alex', 'agaynor'],
     'David Schneider': ['bivab', 'david'],
     'Christian Tismer': ['chris', 'christian', 'tismer',
@@ -41,7 +42,7 @@
     'Mark Pearse': ['mwp'],
     'Toon Verwaest': ['tverwaes'],
     'Eric van Riet Paap': ['ericvrp'],
-    'Jacob Hallen': ['jacob', 'jakob'],
+    'Jacob Hallen': ['jacob', 'jakob', 'jacob hallen'],
     'Anders Lehmann': ['ale', 'anders'],
     'Bert Freudenberg': ['bert'],
     'Boris Feigin': ['boris', 'boria'],
@@ -69,19 +70,25 @@
     'Manuel Jacob': ['mjacob'],
     'Rami Chowdhury': ['necaris'],
     'Stanislaw Halik': ['Stanislaw Halik', 'w31rd0'],
-    'Wenzhu Man':['wenzhu man', 'wenzhuman'],
-    'Anton Gulenko':['anton gulenko', 'anton_gulenko'],
-    'Richard Lancaster':['richardlancaster'],
-    'William Leslie':['William ML Leslie'],
-    'Spenser Bauman':['Spenser Andrew Bauman'],
-    'Raffael Tfirst':['raffael.tfirst at gmail.com'],
-    'timo':['timo at eistee.fritz.box'],
-    'Jasper Schulz':['Jasper.Schulz', 'jbs'],
-    'Aaron Gallagher':['"Aaron Gallagher'],
-    'Yasir Suhail':['yasirs'],
+    'Wenzhu Man': ['wenzhu man', 'wenzhuman'],
+    'Anton Gulenko': ['anton gulenko', 'anton_gulenko'],
+    'Richard Lancaster': ['richardlancaster'],
+    'William Leslie': ['William ML Leslie'],
+    'Spenser Bauman': ['Spenser Andrew Bauman'],
+    'Raffael Tfirst': ['raffael.tfirst at gmail.com'],
+    'timo': ['timo at eistee.fritz.box'],
+    'Jasper Schulz': ['Jasper.Schulz', 'jbs'],
+    'Aaron Gallagher': ['"Aaron Gallagher'],
+    'Yasir Suhail': ['yasirs'],
     'Squeaky': ['squeaky'],
-    "Amaury Forgeot d'Arc": ['amauryfa at gmail.com'],
     "Dodan Mihai": ['mihai.dodan at gmail.com'],
+    'Wim Lavrijsen': ['wlav'],
+    'Toon Verwaest': ['toon', 'tverwaes'],
+    'Seo Sanghyeon': ['sanxiyn'],
+    'Leonardo Santagada': ['santagada'],
+    'Laurence Tratt': ['ltratt'],
+    'Pieter Zieschang': ['pzieschang', 'p_zieschang at yahoo.de'],
+    'John Witulski': ['witulski'],
     }
 
 alias_map = {}
@@ -103,7 +110,8 @@
         return set()
     ignore_words = ['around', 'consulting', 'yesterday', 'for a bit', 'thanks',
                     'in-progress', 'bits of', 'even a little', 'floating',
-                    'a bit', 'reviewing']
+                    'a bit', 'reviewing', 'looking', 'advising', 'partly', 'ish',
+                    'watching', 'mostly', 'jumping']
     sep_words = ['and', ';', '+', '/', 'with special  by']
     nicknames = match.group(1)
     for word in ignore_words:
diff --git a/pypy/module/__pypy__/__init__.py b/pypy/module/__pypy__/__init__.py
--- a/pypy/module/__pypy__/__init__.py
+++ b/pypy/module/__pypy__/__init__.py
@@ -48,6 +48,7 @@
         'int_lshift':      'interp_intop.int_lshift',
         'int_rshift':      'interp_intop.int_rshift',
         'uint_rshift':     'interp_intop.uint_rshift',
+        'int_mulmod':      'interp_intop.int_mulmod',
     }
 
 
diff --git a/pypy/module/__pypy__/interp_intop.py b/pypy/module/__pypy__/interp_intop.py
--- a/pypy/module/__pypy__/interp_intop.py
+++ b/pypy/module/__pypy__/interp_intop.py
@@ -2,7 +2,7 @@
 from rpython.rtyper.lltypesystem import lltype
 from rpython.rtyper.lltypesystem.lloperation import llop
 from rpython.rlib.rarithmetic import r_uint, intmask
-from rpython.rlib.rarithmetic import int_c_div, int_c_mod
+from rpython.rlib.rarithmetic import int_c_div, int_c_mod, mulmod
 from rpython.rlib import jit
 
 
@@ -39,3 +39,7 @@
     n = r_uint(n)
     x = llop.uint_rshift(lltype.Unsigned, n, m)
     return space.newint(intmask(x))
+
+ at unwrap_spec(a=int, b=int, c=int)
+def int_mulmod(space, a, b, c):
+    return space.newint(mulmod(a, b, c))
diff --git a/pypy/module/__pypy__/test/test_intop.py b/pypy/module/__pypy__/test/test_intop.py
--- a/pypy/module/__pypy__/test/test_intop.py
+++ b/pypy/module/__pypy__/test/test_intop.py
@@ -102,3 +102,7 @@
         assert intop.uint_rshift(-1, 1) == sys.maxsize
         assert intop.uint_rshift(-1, bits-2) == 3
         assert intop.uint_rshift(-1, bits-1) == 1
+
+    def test_mulmod(self):
+        from __pypy__ import intop
+        assert intop.int_mulmod(9373891, 9832739, 2**31-1) == 1025488209
diff --git a/pypy/module/_cffi_backend/__init__.py b/pypy/module/_cffi_backend/__init__.py
--- a/pypy/module/_cffi_backend/__init__.py
+++ b/pypy/module/_cffi_backend/__init__.py
@@ -3,7 +3,7 @@
 from rpython.rlib import rdynload, clibffi
 from rpython.rtyper.lltypesystem import rffi
 
-VERSION = "1.11.5"
+VERSION = "1.12.0"
 
 FFI_DEFAULT_ABI = clibffi.FFI_DEFAULT_ABI
 try:
diff --git a/pypy/module/_cffi_backend/newtype.py b/pypy/module/_cffi_backend/newtype.py
--- a/pypy/module/_cffi_backend/newtype.py
+++ b/pypy/module/_cffi_backend/newtype.py
@@ -258,6 +258,11 @@
 SF_PACKED             = 0x08
 SF_STD_FIELD_POS      = 0x80
 
+if sys.platform == 'win32':
+    SF_DEFAULT_PACKING = 8
+else:
+    SF_DEFAULT_PACKING = 0x40000000    # a huge power of two
+
 
 if sys.platform == 'win32':
     DEFAULT_SFLAGS_PLATFORM = SF_MSVC_BITFIELDS
@@ -309,10 +314,18 @@
         w_ctype._custom_field_pos = True
 
 @unwrap_spec(w_ctype=ctypeobj.W_CType, totalsize=int, totalalignment=int,
-             sflags=int)
+             sflags=int, pack=int)
 def complete_struct_or_union(space, w_ctype, w_fields, w_ignored=None,
-                             totalsize=-1, totalalignment=-1, sflags=0):
+                             totalsize=-1, totalalignment=-1, sflags=0,
+                             pack=0):
     sflags = complete_sflags(sflags)
+    if sflags & SF_PACKED:
+        pack = 1
+    elif pack <= 0:
+        pack = SF_DEFAULT_PACKING
+    else:
+        sflags |= SF_PACKED
+
     if (not isinstance(w_ctype, ctypestruct.W_CTypeStructOrUnion)
             or w_ctype.size >= 0):
         raise oefmt(space.w_TypeError,
@@ -362,7 +375,7 @@
         # update the total alignment requirement, but skip it if the
         # field is an anonymous bitfield or if SF_PACKED
         falignorg = ftype.alignof()
-        falign = 1 if sflags & SF_PACKED else falignorg
+        falign = min(pack, falignorg)
         do_align = True
         if (sflags & SF_GCC_ARM_BITFIELDS) == 0 and fbitsize >= 0:
             if (sflags & SF_MSVC_BITFIELDS) == 0:
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -1,7 +1,7 @@
 # ____________________________________________________________
 
 import sys
-assert __version__ == "1.11.5", ("This test_c.py file is for testing a version"
+assert __version__ == "1.12.0", ("This test_c.py file is for testing a version"
                                  " of cffi that differs from the one that we"
                                  " get from 'import _cffi_backend'")
 if sys.version_info < (3,):
@@ -3541,30 +3541,50 @@
     BLong = new_primitive_type("long")
     BChar = new_primitive_type("char")
     BShort = new_primitive_type("short")
-    BStruct = new_struct_type("struct foo")
-    complete_struct_or_union(BStruct, [('a1', BLong, -1),
-                                       ('a2', BChar, -1),
-                                       ('a3', BShort, -1)],
-                             None, -1, -1, SF_PACKED)
-    d = BStruct.fields
-    assert len(d) == 3
-    assert d[0][0] == 'a1'
-    assert d[0][1].type is BLong
+    for extra_args in [(SF_PACKED,), (0, 1)]:
+        BStruct = new_struct_type("struct foo")
+        complete_struct_or_union(BStruct, [('a1', BLong, -1),
+                                           ('a2', BChar, -1),
+                                           ('a3', BShort, -1)],
+                                 None, -1, -1, *extra_args)
+        d = BStruct.fields
+        assert len(d) == 3
+        assert d[0][0] == 'a1'
+        assert d[0][1].type is BLong
+        assert d[0][1].offset == 0
+        assert d[0][1].bitshift == -1
+        assert d[0][1].bitsize == -1
+        assert d[1][0] == 'a2'
+        assert d[1][1].type is BChar
+        assert d[1][1].offset == sizeof(BLong)
+        assert d[1][1].bitshift == -1
+        assert d[1][1].bitsize == -1
+        assert d[2][0] == 'a3'
+        assert d[2][1].type is BShort
+        assert d[2][1].offset == sizeof(BLong) + sizeof(BChar)
+        assert d[2][1].bitshift == -1
+        assert d[2][1].bitsize == -1
+        assert sizeof(BStruct) == sizeof(BLong) + sizeof(BChar) + sizeof(BShort)
+        assert alignof(BStruct) == 1
+    #
+    BStruct2 = new_struct_type("struct foo")
+    complete_struct_or_union(BStruct2, [('b1', BChar, -1),
+                                        ('b2', BLong, -1)],
+                             None, -1, -1, 0, 2)
+    d = BStruct2.fields
+    assert len(d) == 2
+    assert d[0][0] == 'b1'
+    assert d[0][1].type is BChar
     assert d[0][1].offset == 0
     assert d[0][1].bitshift == -1
     assert d[0][1].bitsize == -1
-    assert d[1][0] == 'a2'
-    assert d[1][1].type is BChar
-    assert d[1][1].offset == sizeof(BLong)
+    assert d[1][0] == 'b2'
+    assert d[1][1].type is BLong
+    assert d[1][1].offset == 2
     assert d[1][1].bitshift == -1
     assert d[1][1].bitsize == -1
-    assert d[2][0] == 'a3'
-    assert d[2][1].type is BShort
-    assert d[2][1].offset == sizeof(BLong) + sizeof(BChar)
-    assert d[2][1].bitshift == -1
-    assert d[2][1].bitsize == -1
-    assert sizeof(BStruct) == sizeof(BLong) + sizeof(BChar) + sizeof(BShort)
-    assert alignof(BStruct) == 1
+    assert sizeof(BStruct2) == 2 + sizeof(BLong)
+    assert alignof(BStruct2) == 2
 
 def test_packed_with_bitfields():
     if sys.platform == "win32":
@@ -3915,7 +3935,7 @@
 
 def test_char_pointer_conversion():
     import warnings
-    assert __version__.startswith(("1.8", "1.9", "1.10", "1.11")), (
+    assert __version__.startswith(("1.8", "1.9", "1.10", "1.11", "1.12")), (
         "consider turning the warning into an error")
     BCharP = new_pointer_type(new_primitive_type("char"))
     BIntP = new_pointer_type(new_primitive_type("int"))
diff --git a/pypy/module/_cppyy/converter.py b/pypy/module/_cppyy/converter.py
--- a/pypy/module/_cppyy/converter.py
+++ b/pypy/module/_cppyy/converter.py
@@ -82,10 +82,9 @@
 
 
 class TypeConverter(object):
-    _immutable_fields_ = ['cffi_name', 'uses_local', 'name']
+    _immutable_fields_ = ['cffi_name', 'name']
 
     cffi_name  = None
-    uses_local = False
     name       = ""
 
     def __init__(self, space, extra):
@@ -108,10 +107,10 @@
         from pypy.module._cppyy.interp_cppyy import FastCallNotPossible
         raise FastCallNotPossible
 
-    def convert_argument(self, space, w_obj, address, call_local):
+    def convert_argument(self, space, w_obj, address):
         self._is_abstract(space)
 
-    def convert_argument_libffi(self, space, w_obj, address, call_local):
+    def convert_argument_libffi(self, space, w_obj, address, scratch):
         from pypy.module._cppyy.interp_cppyy import FastCallNotPossible
         raise FastCallNotPossible
 
@@ -125,10 +124,10 @@
     def to_memory(self, space, w_obj, w_value, offset):
         self._is_abstract(space)
 
-    def finalize_call(self, space, w_obj, call_local):
+    def finalize_call(self, space, w_obj):
         pass
 
-    def free_argument(self, space, arg, call_local):
+    def free_argument(self, space, arg):
         pass
 
 
@@ -172,7 +171,7 @@
         state = space.fromcache(ffitypes.State)
         return state.c_voidp
 
-    def convert_argument(self, space, w_obj, address, call_local):
+    def convert_argument(self, space, w_obj, address):
         w_tc = space.findattr(w_obj, space.newtext('typecode'))
         if w_tc is not None and space.text_w(w_tc) != self.typecode:
             raise oefmt(space.w_TypeError,
@@ -208,7 +207,7 @@
 class NumericTypeConverterMixin(object):
     _mixin_ = True
 
-    def convert_argument_libffi(self, space, w_obj, address, call_local):
+    def convert_argument_libffi(self, space, w_obj, address, scratch):
         x = rffi.cast(self.c_ptrtype, address)
         x[0] = self._unwrap_object(space, w_obj)
 
@@ -228,26 +227,23 @@
 
 class ConstRefNumericTypeConverterMixin(NumericTypeConverterMixin):
     _mixin_ = True
-    _immutable_fields_ = ['uses_local']
-
-    uses_local = True
 
     def cffi_type(self, space):
         state = space.fromcache(ffitypes.State)
         return state.c_voidp
 
-    def convert_argument_libffi(self, space, w_obj, address, call_local):
-        assert rffi.sizeof(self.c_type) <= 2*rffi.sizeof(rffi.VOIDP)  # see interp_cppyy.py
+    def convert_argument_libffi(self, space, w_obj, address, scratch):
         obj = self._unwrap_object(space, w_obj)
-        typed_buf = rffi.cast(self.c_ptrtype, call_local)
+        typed_buf = rffi.cast(self.c_ptrtype, scratch)
         typed_buf[0] = obj
         x = rffi.cast(rffi.VOIDPP, address)
-        x[0] = call_local
+        x[0] = scratch
+
 
 class IntTypeConverterMixin(NumericTypeConverterMixin):
     _mixin_ = True
 
-    def convert_argument(self, space, w_obj, address, call_local):
+    def convert_argument(self, space, w_obj, address):
         x = rffi.cast(self.c_ptrtype, address)
         x[0] = self._unwrap_object(space, w_obj)
         ba = rffi.cast(rffi.CCHARP, address)
@@ -256,7 +252,7 @@
 class FloatTypeConverterMixin(NumericTypeConverterMixin):
     _mixin_ = True
 
-    def convert_argument(self, space, w_obj, address, call_local):
+    def convert_argument(self, space, w_obj, address):
         x = rffi.cast(self.c_ptrtype, address)
         x[0] = self._unwrap_object(space, w_obj)
         ba = rffi.cast(rffi.CCHARP, address)
@@ -273,18 +269,18 @@
         state = space.fromcache(ffitypes.State)
         return state.c_void
 
-    def convert_argument(self, space, w_obj, address, call_local):
+    def convert_argument(self, space, w_obj, address):
         self._is_abstract(space)
 
 
 class BoolConverter(ffitypes.typeid(bool), TypeConverter):
-    def convert_argument(self, space, w_obj, address, call_local):
+    def convert_argument(self, space, w_obj, address):
         x = rffi.cast(rffi.LONGP, address)
         x[0] = self._unwrap_object(space, w_obj)
         ba = rffi.cast(rffi.CCHARP, address)
         ba[capi.c_function_arg_typeoffset(space)] = 'b'
 
-    def convert_argument_libffi(self, space, w_obj, address, call_local):
+    def convert_argument_libffi(self, space, w_obj, address, scratch):
         x = rffi.cast(rffi.LONGP, address)
         x[0] = self._unwrap_object(space, w_obj)
 
@@ -303,13 +299,13 @@
             address[0] = '\x00'
 
 class CharConverter(ffitypes.typeid(rffi.CHAR), TypeConverter):
-    def convert_argument(self, space, w_obj, address, call_local):
+    def convert_argument(self, space, w_obj, address):
         x = rffi.cast(rffi.CCHARP, address)
         x[0] = self._unwrap_object(space, w_obj)
         ba = rffi.cast(rffi.CCHARP, address)
         ba[capi.c_function_arg_typeoffset(space)] = 'b'
 
-    def convert_argument_libffi(self, space, w_obj, address, call_local):
+    def convert_argument_libffi(self, space, w_obj, address, scratch):
         x = rffi.cast(self.c_ptrtype, address)
         x[0] = self._unwrap_object(space, w_obj)
 
@@ -348,7 +344,7 @@
         state = space.fromcache(ffitypes.State)
         return state.c_voidp
 
-    def convert_argument_libffi(self, space, w_obj, address, call_local):
+    def convert_argument_libffi(self, space, w_obj, address, scratch):
         from pypy.module._cppyy.interp_cppyy import FastCallNotPossible
         raise FastCallNotPossible
 
@@ -381,7 +377,7 @@
 
 
 class CStringConverter(TypeConverter):
-    def convert_argument(self, space, w_obj, address, call_local):
+    def convert_argument(self, space, w_obj, address):
         x = rffi.cast(rffi.LONGP, address)
         arg = space.text_w(w_obj)
         x[0] = rffi.cast(rffi.LONG, rffi.str2charp(arg))
@@ -393,7 +389,7 @@
         charpptr = rffi.cast(rffi.CCHARPP, address)
         return space.newtext(rffi.charp2str(charpptr[0]))
 
-    def free_argument(self, space, arg, call_local):
+    def free_argument(self, space, arg):
         lltype.free(rffi.cast(rffi.CCHARPP, arg)[0], flavor='raw')
 
 class CStringConverterWithSize(CStringConverter):
@@ -423,13 +419,13 @@
         state = space.fromcache(ffitypes.State)
         return state.c_voidp
 
-    def convert_argument(self, space, w_obj, address, call_local):
+    def convert_argument(self, space, w_obj, address):
         x = rffi.cast(rffi.VOIDPP, address)
         x[0] = self._unwrap_object(space, w_obj)
         ba = rffi.cast(rffi.CCHARP, address)
         ba[capi.c_function_arg_typeoffset(space)] = 'o'
 
-    def convert_argument_libffi(self, space, w_obj, address, call_local):
+    def convert_argument_libffi(self, space, w_obj, address, scratch):
         x = rffi.cast(rffi.VOIDPP, address)
         x[0] = self._unwrap_object(space, w_obj)
 
@@ -452,37 +448,39 @@
             address[0] = rffi.cast(rffi.VOIDP, self._unwrap_object(space, w_value))
 
 class VoidPtrPtrConverter(TypeConverter):
-    _immutable_fields_ = ['uses_local', 'typecode']
+    typecode = 'p'
 
-    uses_local = True
-    typecode = 'a'
+    def __init__(self, space, extra):
+        self.ref_buffer = lltype.nullptr(rffi.VOIDPP.TO)
 
-    def convert_argument(self, space, w_obj, address, call_local):
+    def convert_argument(self, space, w_obj, address):
         x = rffi.cast(rffi.VOIDPP, address)
-        ba = rffi.cast(rffi.CCHARP, address)
         try:
             x[0] = get_rawbuffer(space, w_obj)
         except TypeError:
-            r = rffi.cast(rffi.VOIDPP, call_local)
-            r[0] = rffi.cast(rffi.VOIDP, get_rawobject(space, w_obj))
-            x[0] = rffi.cast(rffi.VOIDP, call_local)
+            ptr = rffi.cast(rffi.VOIDP, get_rawobject(space, w_obj))
+            self.ref_buffer = lltype.malloc(rffi.VOIDPP.TO, 1, flavor='raw')
+            self.ref_buffer[0] = ptr
+            x[0] = self.ref_buffer
+        ba = rffi.cast(rffi.CCHARP, address)
         ba[capi.c_function_arg_typeoffset(space)] = self.typecode
 
-    def finalize_call(self, space, w_obj, call_local):
-        r = rffi.cast(rffi.VOIDPP, call_local)
-        try:
-            set_rawobject(space, w_obj, r[0])
-        except OperationError:
-            pass             # no set on buffer/array/None
+    def finalize_call(self, space, w_obj):
+        if self.ref_buffer:
+            set_rawobject(space, w_obj, self.ref_buffer[0])
+
+    def free_argument(self, space, arg):
+        if self.ref_buffer:
+            lltype.free(self.ref_buffer, flavor='raw')
+            self.ref_buffer = lltype.nullptr(rffi.VOIDPP.TO)
 
 class VoidPtrRefConverter(VoidPtrPtrConverter):
-    _immutable_fields_ = ['uses_local', 'typecode']
-    uses_local = True
+    _immutable_fields_ = ['typecode']
     typecode   = 'V'
 
 class InstanceRefConverter(TypeConverter):
     _immutable_fields_ = ['typecode', 'clsdecl']
-    typecode    = 'V'
+    typecode = 'V'
 
     def __init__(self, space, clsdecl):
         from pypy.module._cppyy.interp_cppyy import W_CPPClassDecl
@@ -508,14 +506,14 @@
         state = space.fromcache(ffitypes.State)
         return state.c_voidp
 
-    def convert_argument(self, space, w_obj, address, call_local):
+    def convert_argument(self, space, w_obj, address):
         x = rffi.cast(rffi.VOIDPP, address)
         x[0] = rffi.cast(rffi.VOIDP, self._unwrap_object(space, w_obj))
         address = rffi.cast(capi.C_OBJECT, address)
         ba = rffi.cast(rffi.CCHARP, address)
         ba[capi.c_function_arg_typeoffset(space)] = self.typecode
 
-    def convert_argument_libffi(self, space, w_obj, address, call_local):
+    def convert_argument_libffi(self, space, w_obj, address, scratch):
         x = rffi.cast(rffi.VOIDPP, address)
         x[0] = rffi.cast(rffi.VOIDP, self._unwrap_object(space, w_obj))
 
@@ -539,7 +537,7 @@
 
 class InstanceConverter(InstanceRefConverter):
 
-    def convert_argument_libffi(self, space, w_obj, address, call_local):
+    def convert_argument_libffi(self, space, w_obj, address, scratch):
         from pypy.module._cppyy.interp_cppyy import FastCallNotPossible
         raise FastCallNotPossible       # TODO: by-value is a jit_libffi special case
 
@@ -551,9 +549,8 @@
     def to_memory(self, space, w_obj, w_value, offset):
         self._is_abstract(space)
 
-
 class InstancePtrConverter(InstanceRefConverter):
-    typecode    = 'o'
+    typecode = 'o'
 
     def _unwrap_object(self, space, w_obj):
         try:
@@ -574,36 +571,41 @@
         address[0] = rffi.cast(rffi.VOIDP, self._unwrap_object(space, w_value))
 
 class InstancePtrPtrConverter(InstancePtrConverter):
-    _immutable_fields_ = ['uses_local']
+    typecode = 'o'
 
-    uses_local = True
+    def __init__(self, space, extra):
+        InstancePtrConverter.__init__(self, space, extra)
+        self.ref_buffer = lltype.nullptr(rffi.VOIDPP.TO)
 
-    def convert_argument(self, space, w_obj, address, call_local):
-        r = rffi.cast(rffi.VOIDPP, call_local)
-        r[0] = rffi.cast(rffi.VOIDP, self._unwrap_object(space, w_obj))
+    def convert_argument(self, space, w_obj, address):
         x = rffi.cast(rffi.VOIDPP, address)
-        x[0] = rffi.cast(rffi.VOIDP, call_local)
-        address = rffi.cast(capi.C_OBJECT, address)
+        ptr = rffi.cast(rffi.VOIDP, self._unwrap_object(space, w_obj))
+        self.ref_buffer = lltype.malloc(rffi.VOIDPP.TO, 1, flavor='raw')
+        self.ref_buffer[0] = ptr
+        x[0] = self.ref_buffer
         ba = rffi.cast(rffi.CCHARP, address)
-        ba[capi.c_function_arg_typeoffset(space)] = 'o'
+        ba[capi.c_function_arg_typeoffset(space)] = self.typecode
 
-    def convert_argument_libffi(self, space, w_obj, address, call_local):
+    def convert_argument_libffi(self, space, w_obj, address, scratch):
         # TODO: finalize_call not yet called for fast call (see interp_cppyy.py)
         from pypy.module._cppyy.interp_cppyy import FastCallNotPossible
         raise FastCallNotPossible
 
-    def finalize_call(self, space, w_obj, call_local):
-        from pypy.module._cppyy.interp_cppyy import W_CPPInstance
-        assert isinstance(w_obj, W_CPPInstance)
-        r = rffi.cast(rffi.VOIDPP, call_local)
-        w_obj._rawobject = rffi.cast(capi.C_OBJECT, r[0])
-
     def from_memory(self, space, w_obj, w_pycppclass, offset):
         address = rffi.cast(capi.C_OBJECT, self._get_raw_address(space, w_obj, offset))
         from pypy.module._cppyy import interp_cppyy
         return interp_cppyy.wrap_cppinstance(
             space, address, self.clsdecl, do_cast=False, is_ref=True)
 
+    def finalize_call(self, space, w_obj):
+        if self.ref_buffer:
+            set_rawobject(space, w_obj, self.ref_buffer[0])
+
+    def free_argument(self, space, arg):
+        if self.ref_buffer:
+            lltype.free(self.ref_buffer, flavor='raw')
+            self.ref_buffer = lltype.nullptr(rffi.VOIDPP.TO)
+
 class StdStringConverter(InstanceConverter):
 
     def __init__(self, space, extra):
@@ -628,7 +630,7 @@
         except Exception:
             InstanceConverter.to_memory(self, space, w_obj, w_value, offset)
 
-    def free_argument(self, space, arg, call_local):
+    def free_argument(self, space, arg):
         capi.c_destruct(space, self.clsdecl, rffi.cast(capi.C_OBJECT, rffi.cast(rffi.VOIDPP, arg)[0]))
 
 class StdStringRefConverter(InstancePtrConverter):
@@ -646,7 +648,7 @@
         state = space.fromcache(ffitypes.State)
         return state.c_voidp
 
-    def convert_argument(self, space, w_obj, address, call_local):
+    def convert_argument(self, space, w_obj, address):
         if hasattr(space, "fake"):
             raise NotImplementedError
         space.getbuiltinmodule("cpyext")
@@ -657,7 +659,7 @@
         ba = rffi.cast(rffi.CCHARP, address)
         ba[capi.c_function_arg_typeoffset(space)] = 'a'
 
-    def convert_argument_libffi(self, space, w_obj, address, call_local):
+    def convert_argument_libffi(self, space, w_obj, address, scratch):
         # TODO: free_argument not yet called for fast call (see interp_cppyy.py)
         from pypy.module._cppyy.interp_cppyy import FastCallNotPossible
         raise FastCallNotPossible
@@ -671,7 +673,7 @@
         x = rffi.cast(rffi.VOIDPP, address)
         x[0] = rffi.cast(rffi.VOIDP, ref)"""
 
-    def free_argument(self, space, arg, call_local):
+    def free_argument(self, space, arg):
         if hasattr(space, "fake"):
             raise NotImplementedError
         space.getbuiltinmodule("cpyext")
@@ -685,7 +687,7 @@
     def __init__(self, space, signature):
         self.signature = signature
 
-    def convert_argument(self, space, w_obj, address, call_local):
+    def convert_argument(self, space, w_obj, address):
         # TODO: atm, does not actually get an overload, but a staticmethod
         from pypy.module._cppyy.interp_cppyy import W_CPPOverload
         cppol = space.interp_w(W_CPPOverload, w_obj)
@@ -740,7 +742,7 @@
         raise oefmt(space.w_TypeError,
                     "cannot pass %T instance as %s", w_obj, self.rawdecl.name)
 
-    def convert_argument(self, space, w_obj, address, call_local):
+    def convert_argument(self, space, w_obj, address):
         x = rffi.cast(rffi.VOIDPP, address)
         x[0] = rffi.cast(rffi.VOIDP, self._unwrap_object(space, w_obj))
         address = rffi.cast(capi.C_OBJECT, address)
diff --git a/pypy/module/_cppyy/interp_cppyy.py b/pypy/module/_cppyy/interp_cppyy.py
--- a/pypy/module/_cppyy/interp_cppyy.py
+++ b/pypy/module/_cppyy/interp_cppyy.py
@@ -1,6 +1,9 @@
 import pypy.module._cppyy.capi as capi
 
 from pypy.interpreter.error import OperationError, oefmt
+from pypy.interpreter.function import Method
+from pypy.interpreter.argument import Arguments
+from pypy.interpreter.typedef import interp_attrproperty_w, descr_generic_ne, make_weakref_descr
 from pypy.interpreter.gateway import interp2app, unwrap_spec
 from pypy.interpreter.typedef import TypeDef, GetSetProperty, interp_attrproperty
 from pypy.interpreter.baseobjspace import W_Root
@@ -166,11 +169,13 @@
 #  W_CPPConstructorOverload:      constructors
 #  W_CPPStaticOverload:           free and static functions
 #  W_CPPTemplateOverload:         templated methods
-#  W_CPPTemplateStaticOveload:    templated free and static functions
+#  W_CPPTemplateStaticOverload:   templated free and static functions
 #
 #  CPPMethod:         a single function or method (base class)
 #  CPPSetItem:        specialization for Python's __setitem__
 #
+#  MethodWithProps:   python instancemethod that forwards properties
+#
 # All methods/functions derive from CPPMethod and are collected as overload
 # candidates in user-facing overload classes. Templated methods are a two-step
 # process, where first the template is instantiated (or selected if already
@@ -183,13 +188,13 @@
     the memory_regulator."""
 
     _attrs_ = ['space', 'scope', 'cppmethod', 'arg_defs', 'args_required',
-               'converters', 'executor', '_funcaddr', 'cif_descr', 'uses_local']
+               'converters', 'executor', '_funcaddr', 'cif_descr']
     _immutable_fields_ = ['scope', 'cppmethod', 'arg_defs', 'args_required',
-                          'converters', 'executor', 'uses_local']
+               'converters', 'executor', '_funcaddr', 'cif_descr']
 
-    def __init__(self, space, declaring_scope, cppmethod, arg_defs, args_required):
+    def __init__(self, space, decl_scope, cppmethod, arg_defs, args_required):
         self.space = space
-        self.scope = declaring_scope
+        self.scope = decl_scope
         self.cppmethod = cppmethod
         self.arg_defs = arg_defs
         self.args_required = args_required
@@ -200,14 +205,6 @@
         self.executor = None
         self.cif_descr = lltype.nullptr(jit_libffi.CIF_DESCRIPTION)
         self._funcaddr = lltype.nullptr(capi.C_FUNC_PTR.TO)
-        self.uses_local = False
-
-    def _address_from_local_buffer(self, call_local, idx):
-        if not call_local:
-            return call_local
-        stride = 2*rffi.sizeof(rffi.VOIDP)
-        loc_idx = lltype.direct_ptradd(rffi.cast(rffi.CCHARP, call_local), idx*stride)
-        return rffi.cast(rffi.VOIDP, loc_idx)
 
     @jit.unroll_safe
     def call(self, cppthis, args_w, useffi):
@@ -233,61 +230,55 @@
             except Exception:
                 pass
 
-        # some calls, e.g. for ptr-ptr or reference need a local array to store data for
-        # the duration of the call
-        if self.uses_local:
-            call_local = lltype.malloc(rffi.VOIDP.TO, 2*len(args_w), flavor='raw')
-        else:
-            call_local = lltype.nullptr(rffi.VOIDP.TO)
+        # attempt to call directly through ffi chain
+        if useffi and self._funcaddr:
+            try:
+                return self.do_fast_call(cppthis, args_w)
+            except FastCallNotPossible:
+                pass      # can happen if converters or executor does not implement ffi
 
+        # ffi chain must have failed; using stub functions instead
+        args, stat = self.prepare_arguments(args_w)
         try:
-            # attempt to call directly through ffi chain
-            if useffi and self._funcaddr:
-                try:
-                    return self.do_fast_call(cppthis, args_w, call_local)
-                except FastCallNotPossible:
-                    pass      # can happen if converters or executor does not implement ffi
-
-            # ffi chain must have failed; using stub functions instead
-            args, stat = self.prepare_arguments(args_w, call_local)
-            try:
-                result = self.executor.execute(
-                    self.space, self.cppmethod, cppthis, len(args_w), args)
-                if stat[0] != rffi.cast(rffi.ULONG, 0):
-                    what = rffi.cast(rffi.CCHARP, stat[1])
-                    pywhat = rffi.charp2str(what)
-                    capi.c_free(self.space, rffi.cast(rffi.VOIDP, what))
-                    raise OperationError(self.space.w_Exception, self.space.newtext(pywhat))
-                return result
-            finally:
-                self.finalize_call(args, args_w, call_local)
+            result = self.executor.execute(
+                self.space, self.cppmethod, cppthis, len(args_w), args)
+            if stat[0] != rffi.cast(rffi.ULONG, 0):
+                what = rffi.cast(rffi.CCHARP, stat[1])
+                pywhat = rffi.charp2str(what)
+                capi.c_free(self.space, rffi.cast(rffi.VOIDP, what))
+                raise OperationError(self.space.w_Exception, self.space.newtext(pywhat))
+            return result
         finally:
-            if call_local:
-                lltype.free(call_local, flavor='raw')
+            self.finalize_call(args, args_w)
 
     @jit.unroll_safe
-    def do_fast_call(self, cppthis, args_w, call_local):
+    def do_fast_call(self, cppthis, args_w):
         if self.cif_descr == lltype.nullptr(jit_libffi.CIF_DESCRIPTION):
             raise FastCallNotPossible
         jit.promote(self)
         cif_descr = self.cif_descr
-        buffer = lltype.malloc(rffi.CCHARP.TO, cif_descr.exchange_size, flavor='raw')
+        # add extra space for const-ref support (see converter.py)
+        buffer = lltype.malloc(rffi.CCHARP.TO, cif_descr.exchange_size+len(self.arg_defs)*rffi.sizeof(rffi.DOUBLE), flavor='raw')
+        thisoff = 0
         try:
-            # this pointer
-            data = rffi.ptradd(buffer, cif_descr.exchange_args[0])
-            x = rffi.cast(rffi.LONGP, data)       # LONGP needed for test_zjit.py
-            x[0] = rffi.cast(rffi.LONG, cppthis)
+            if cppthis:
+                # this pointer
+                data = rffi.ptradd(buffer, cif_descr.exchange_args[0])
+                x = rffi.cast(rffi.LONGP, data)       # LONGP needed for test_zjit.py
+                x[0] = rffi.cast(rffi.LONG, cppthis)
+                thisoff = 1
 
-            # other arguments and defaults
-            i = len(self.arg_defs) + 1
+            # actual provided arguments
+            i = -1      # needed if all arguments are defaults
             for i in range(len(args_w)):
                 conv = self.converters[i]
-                w_arg = args_w[i]
-                data = rffi.ptradd(buffer, cif_descr.exchange_args[i+1])
-                conv.convert_argument_libffi(self.space, w_arg, data, call_local)
+                data = rffi.ptradd(buffer, cif_descr.exchange_args[i+thisoff])
+                scratch = rffi.ptradd(buffer, cif_descr.exchange_size+i*rffi.sizeof(rffi.DOUBLE))
+                conv.convert_argument_libffi(self.space, args_w[i], data, scratch)
+            # drop in defaults for the rest
             for j in range(i+1, len(self.arg_defs)):
                 conv = self.converters[j]
-                data = rffi.ptradd(buffer, cif_descr.exchange_args[j+1])
+                data = rffi.ptradd(buffer, cif_descr.exchange_args[j+thisoff])
                 conv.default_argument_libffi(self.space, data)
 
             assert self._funcaddr
@@ -346,22 +337,17 @@
         self.executor = executor.get_executor(
             self.space, capi.c_method_result_type(self.space, self.cppmethod))
 
-        for conv in self.converters:
-            if conv.uses_local:
-                self.uses_local = True
-                break
-
         # Each CPPMethod corresponds one-to-one to a C++ equivalent and cppthis
         # has been offset to the matching class. Hence, the libffi pointer is
         # uniquely defined and needs to be setup only once.
         funcaddr = capi.c_function_address(self.space, self.cppmethod)
-        if funcaddr and cppthis:      # TODO: methods only for now
+        if funcaddr:
             state = self.space.fromcache(ffitypes.State)
 
-            # argument type specification (incl. cppthis)
+            # argument type specification (incl. cppthis if applicable)
             fargs = []
             try:
-                fargs.append(state.c_voidp)
+                if cppthis: fargs.append(state.c_voidp)
                 for i, conv in enumerate(self.converters):
                     fargs.append(conv.cffi_type(self.space))
                 fresult = self.executor.cffi_type(self.space)
@@ -386,7 +372,7 @@
             self._funcaddr = funcaddr
 
     @jit.unroll_safe
-    def prepare_arguments(self, args_w, call_local):
+    def prepare_arguments(self, args_w):
         args = capi.c_allocate_function_args(self.space, len(args_w))
         stride = capi.c_function_arg_sizeof(self.space)
         for i in range(len(args_w)):
@@ -394,15 +380,13 @@
             w_arg = args_w[i]
             try:
                 arg_i = lltype.direct_ptradd(rffi.cast(rffi.CCHARP, args), i*stride)
-                loc_i = self._address_from_local_buffer(call_local, i)
-                conv.convert_argument(self.space, w_arg, rffi.cast(capi.C_OBJECT, arg_i), loc_i)
+                conv.convert_argument(self.space, w_arg, rffi.cast(capi.C_OBJECT, arg_i))
             except:
                 # fun :-(
                 for j in range(i):
                     conv = self.converters[j]
                     arg_j = lltype.direct_ptradd(rffi.cast(rffi.CCHARP, args), j*stride)
-                    loc_j = self._address_from_local_buffer(call_local, j)
-                    conv.free_argument(self.space, rffi.cast(capi.C_OBJECT, arg_j), loc_j)
+                    conv.free_argument(self.space, rffi.cast(capi.C_OBJECT, arg_j))
                 capi.c_deallocate_function_args(self.space, args)
                 raise
         stat = rffi.cast(rffi.ULONGP,
@@ -411,14 +395,13 @@
         return args, stat
 
     @jit.unroll_safe
-    def finalize_call(self, args, args_w, call_local):
+    def finalize_call(self, args, args_w):
         stride = capi.c_function_arg_sizeof(self.space)
         for i in range(len(args_w)):
             conv = self.converters[i]
             arg_i = lltype.direct_ptradd(rffi.cast(rffi.CCHARP, args), i*stride)
-            loc_i = self._address_from_local_buffer(call_local, i)
-            conv.finalize_call(self.space, args_w[i], loc_i)
-            conv.free_argument(self.space, rffi.cast(capi.C_OBJECT, arg_i), loc_i)
+            conv.finalize_call(self.space, args_w[i])
+            conv.free_argument(self.space, rffi.cast(capi.C_OBJECT, arg_i))
         capi.c_deallocate_function_args(self.space, args)
 
     def signature(self, show_formalargs=True):
@@ -466,42 +449,81 @@
         CPPMethod.call(self, cppthis, args_w, useffi)
 
 
+# CPPOverloads have settable flags that control memory and ffi behavior. These flags
+# need forwarding, which the normal instancemethod does not provide, hence this
+# derived class.
+class MethodWithProps(Method):
+    # allow user to determine ffi use rules per overload
+    def fget_useffi(self, space):
+        f = space.interp_w(W_CPPOverload, self.w_function)
+        return f.fget_useffi(space)
+
+    @unwrap_spec(value=bool)
+    def fset_useffi(self, space, value):
+        f = space.interp_w(W_CPPOverload, self.w_function)
+        f.fset_useffi(space, value)
+
+MethodWithProps.typedef = TypeDef(
+    "cpp_instancemethod",
+    __doc__ = """cpp_instancemethod(function, instance, class)
+
+Create an instance method object.""",
+    __new__ = interp2app(MethodWithProps.descr_method__new__.im_func),
+    __call__ = interp2app(MethodWithProps.descr_method_call),
+    __get__ = interp2app(MethodWithProps.descr_method_get),
+    im_func = interp_attrproperty_w('w_function', cls=MethodWithProps),
+    __func__ = interp_attrproperty_w('w_function', cls=MethodWithProps),
+    im_self = interp_attrproperty_w('w_instance', cls=MethodWithProps),
+    __self__ = interp_attrproperty_w('w_instance', cls=MethodWithProps),
+    im_class = interp_attrproperty_w('w_class', cls=MethodWithProps),
+    __getattribute__ = interp2app(MethodWithProps.descr_method_getattribute),
+    __eq__ = interp2app(MethodWithProps.descr_method_eq),
+    __ne__ = descr_generic_ne,
+    __hash__ = interp2app(MethodWithProps.descr_method_hash),
+    __repr__ = interp2app(MethodWithProps.descr_method_repr),
+    __reduce__ = interp2app(MethodWithProps.descr_method__reduce__),
+    __weakref__ = make_weakref_descr(MethodWithProps),
+    __useffi__ = GetSetProperty(MethodWithProps.fget_useffi, MethodWithProps.fset_useffi),
+    )
+MethodWithProps.typedef.acceptable_as_base_class = False
+
+
 class W_CPPOverload(W_Root):
     """App-level dispatcher: controls a collection of (potentially) overloaded methods
     or functions. Calls these in order and deals with error handling and reporting."""
 
-    _attrs_ = ['space', 'scope', 'functions', 'flags', 'w_this']
+    _attrs_ = ['space', 'scope', 'functions', 'flags']
     _immutable_fields_ = ['scope', 'functions[*]']
 
-    def __init__(self, space, declaring_scope, functions, flags = OVERLOAD_FLAGS_USE_FFI):
+    def __init__(self, space, decl_scope, funcs, flags = OVERLOAD_FLAGS_USE_FFI):
         self.space  = space
-        self.scope  = declaring_scope
+        self.scope  = decl_scope
         from rpython.rlib import debug
-        self.functions = debug.make_sure_not_resized(functions)
+        self.functions = debug.make_sure_not_resized(funcs)
         self.flags  = flags
-        self.w_this = self.space.w_None
+
+    def descr_get(self, w_obj, w_cls=None):
+        """functionobject.__get__(obj[, type]) -> method"""
+        # TODO: check validity of w_cls if given
+        space = self.space
+        asking_for_bound = (space.is_none(w_cls) or
+                            not space.is_w(w_obj, space.w_None) or
+                            space.is_w(w_cls, space.type(space.w_None)))
+        if asking_for_bound:
+            return MethodWithProps(space, self, w_obj, w_cls)
+        else:
+            return self  # unbound methods don't exist in Python 3, return self
 
     @unwrap_spec(args_w='args_w')
-    def descr_get(self, w_cppinstance, args_w):
-        if self.space.is_w(w_cppinstance, self.space.w_None):
-            return self  # unbound, so no new instance needed
-        cppol = W_CPPOverload(self.space, self.scope, self.functions, self.flags)
-        cppol.w_this = w_cppinstance
-        return cppol     # bound
-
-    @unwrap_spec(args_w='args_w')
-    def call(self, args_w):
-        if self.space.is_w(self.w_this, self.space.w_None) and len(args_w):
-            w_this = args_w[0]
-            args_w = args_w[1:]
-        else:
-            w_this = self.w_this
+    def call_args(self, args_w):
+        jit.promote(self)
+        w_this = args_w[0]
         cppinstance = self.space.interp_w(W_CPPInstance, w_this)
         cppinstance._nullcheck()
         if not capi.c_is_subtype(self.space, cppinstance.clsdecl, self.scope):
             raise oefmt(self.space.w_TypeError,
                 "cannot pass %T instance as %s", w_this, self.scope.name)
-        return self.call_impl(cppinstance.get_cppthis(self.scope), args_w)
+        return self.call_impl(cppinstance.get_cppthis(self.scope), args_w[1:])
 
     @jit.unroll_safe
     def call_impl(self, cppthis, args_w):
@@ -576,13 +598,17 @@
     def fget_doc(self, space):
         return self.prototype()
 
+    def getname(self, space): 
+        # for the benefit of Method/instancemethod
+        return capi.c_method_name(space, self.functions[0].cppmethod)
+
     def __repr__(self):
         return "W_CPPOverload(%s)" % [f.prototype() for f in self.functions]
 
 W_CPPOverload.typedef = TypeDef(
     'CPPOverload',
     __get__    = interp2app(W_CPPOverload.descr_get),
-    __call__   = interp2app(W_CPPOverload.call),
+    __call__   = interp2app(W_CPPOverload.call_args),
     __useffi__ = GetSetProperty(W_CPPOverload.fget_useffi, W_CPPOverload.fset_useffi),
     __doc__    = GetSetProperty(W_CPPOverload.fget_doc)
 )
@@ -593,27 +619,24 @@
 class W_CPPStaticOverload(W_CPPOverload):
     _attrs_ = []
 
-    @unwrap_spec(args_w='args_w')
-    def descr_get(self, w_cppinstance, args_w):
-        if isinstance(w_cppinstance, W_CPPInstance):
+    def descr_get(self, w_obj, w_cls=None):
+        if isinstance(w_obj, W_CPPInstance):
             # two possibilities: this is a static function called on an
             # instance and w_this must not be set, or a free function rebound
             # onto a class and w_this should be set
-            cppinstance = self.space.interp_w(W_CPPInstance, w_cppinstance)
+            cppinstance = self.space.interp_w(W_CPPInstance, w_obj)
             if cppinstance.clsdecl.handle != self.scope.handle:
-                cppol = W_CPPStaticOverload(self.space, self.scope, self.functions, self.flags)
-                cppol.w_this = w_cppinstance
-                return cppol       # bound
+                return MethodWithProps(self.space, self, w_obj, w_cls)    # bound
         return self      # unbound
 
     @unwrap_spec(args_w='args_w')
-    def call(self, args_w):
-        if not self.space.is_w(self.w_this, self.space.w_None):
-            # free function used as bound method, put self back into args_w
-            cppinstance = self.space.interp_w(W_CPPInstance, self.w_this)
-            cppinstance._nullcheck()
-            args_w = [self.w_this] + args_w
+    def call_args(self, args_w):
+        jit.promote(self)
+        #if isinstance(args_w[0], W_CPPInstance):
+            # free function used as bound method, leave in place
         return self.call_impl(capi.C_NULL_OBJECT, args_w)
+        # free functions are implemented as methods of 'namespace' classes, remove 'instance'
+        #return self.call_impl(capi.C_NULL_OBJECT, args_w[1:])
 
     def __repr__(self):
         return "W_CPPStaticOverload(%s)" % [f.prototype() for f in self.functions]
@@ -621,7 +644,7 @@
 W_CPPStaticOverload.typedef = TypeDef(
     'CPPStaticOverload',
     __get__    = interp2app(W_CPPStaticOverload.descr_get),
-    __call__   = interp2app(W_CPPStaticOverload.call),
+    __call__   = interp2app(W_CPPStaticOverload.call_args),
     __useffi__ = GetSetProperty(W_CPPStaticOverload.fget_useffi, W_CPPStaticOverload.fset_useffi),
     __doc__    = GetSetProperty(W_CPPStaticOverload.fget_doc)
 )
@@ -630,27 +653,20 @@
 class W_CPPConstructorOverload(W_CPPOverload):
     _attrs_ = []
 
-    @unwrap_spec(args_w='args_w')
-    def descr_get(self, w_cppinstance, args_w):
-        if self.space.is_w(w_cppinstance, self.space.w_None):
-            return self  # unbound (TODO: probably useless)
-        cppol = W_CPPConstructorOverload(self.space, self.scope, self.functions, self.flags)
-        cppol.w_this = w_cppinstance
-        return cppol     # bound
+    def __init__(self, space, decl_scope, funcs, flags = OVERLOAD_FLAGS_USE_FFI):
+        W_CPPOverload.__init__(self, space, decl_scope, funcs, flags)
+        self.flags &= ~OVERLOAD_FLAGS_USE_FFI
 
     @unwrap_spec(args_w='args_w')
-    def call(self, args_w):
+    def call_args(self, args_w):
+        jit.promote(self)
         # TODO: factor out the following:
         if capi.c_is_abstract(self.space, self.scope.handle):
             raise oefmt(self.space.w_TypeError,
                         "cannot instantiate abstract class '%s'",
                         self.scope.name)
-        if self.space.is_w(self.w_this, self.space.w_None) and len(args_w):
-            cppinstance = self.space.interp_w(W_CPPInstance, args_w[0])
-            args_w = args_w[1:]
-        else:
-            cppinstance = self.space.interp_w(W_CPPInstance, self.w_this)
-        w_result = self.call_impl(rffi.cast(capi.C_OBJECT, self.scope.handle), args_w)
+        cppinstance = self.space.interp_w(W_CPPInstance, args_w[0])
+        w_result = self.call_impl(rffi.cast(capi.C_OBJECT, self.scope.handle), args_w[1:])
         newthis = rffi.cast(capi.C_OBJECT, self.space.uint_w(w_result))
         if cppinstance is not None:
             cppinstance._rawobject = newthis
@@ -662,7 +678,7 @@
 W_CPPConstructorOverload.typedef = TypeDef(
     'CPPConstructorOverload',
     __get__    = interp2app(W_CPPConstructorOverload.descr_get),
-    __call__   = interp2app(W_CPPConstructorOverload.call),
+    __call__   = interp2app(W_CPPConstructorOverload.call_args),
     __doc__    = GetSetProperty(W_CPPConstructorOverload.fget_doc)
 )
 
@@ -725,7 +741,9 @@
         # try to match with run-time instantiations
         for cppol in self.master.overloads.values():
             try:
-                return cppol.descr_get(self.w_this, []).call(args_w)
+                if not self.space.is_w(self.w_this, self.space.w_None):
+                    return self.space.call_obj_args(cppol, self.w_this, Arguments(self.space, args_w))
+                return self.space.call_args(cppol, Arguments(self.space, args_w))
             except Exception:
                 pass    # completely ignore for now; have to see whether errors become confusing
 
@@ -748,7 +766,9 @@
             except KeyError:
                 self.master.overloads[fullname] = method
 
-        return method.descr_get(self.w_this, []).call(args_w)
+        if not self.space.is_w(self.w_this, self.space.w_None):
+            return self.space.call_obj_args(method, self.w_this, Arguments(self.space, args_w))
+        return self.space.call_args(method, Arguments(self.space, args_w))
 
     def getitem_impl(self, name, args_w):
         space = self.space
@@ -771,25 +791,25 @@
             if c_fullname != fullname:
                 self.master.overloads[c_fullname] = method
 
-        return method.descr_get(self.w_this, [])
+        return method.descr_get(self.w_this, None)
 
 
 class W_CPPTemplateOverload(W_CPPOverload, TemplateOverloadMixin):
     """App-level dispatcher to allow both lookup/instantiation of templated methods and
     dispatch among overloads between templated and non-templated method."""
 
-    _attrs_ = ['name', 'overloads', 'master']
+    _attrs_ = ['name', 'overloads', 'master', 'w_this']
     _immutable_fields_ = ['name']
 
-    def __init__(self, space, name, declaring_scope, functions, flags = OVERLOAD_FLAGS_USE_FFI):
-         W_CPPOverload.__init__(self, space, declaring_scope, functions, flags)
+    def __init__(self, space, name, decl_scope, functions, flags = OVERLOAD_FLAGS_USE_FFI):
+         W_CPPOverload.__init__(self, space, decl_scope, functions, flags)
          self.name = name
          self.overloads = {}
          self.master = self
+         self.w_this = space.w_None
 
-    @unwrap_spec(args_w='args_w')
-    def descr_get(self, w_cppinstance, args_w):
-        # like W_CPPOverload, but returns W_CPPTemplateOverload
+    def descr_get(self, w_cppinstance, w_cls=None):
+        # TODO: don't return copy, but bind in an external object (like W_CPPOverload)
         if self.space.is_w(w_cppinstance, self.space.w_None):
             return self  # unbound, so no new instance needed
         cppol = W_CPPTemplateOverload(self.space, self.name, self.scope, self.functions, self.flags)
@@ -798,13 +818,13 @@
         return cppol     # bound
 
     @unwrap_spec(args_w='args_w')
-    def call(self, args_w):
+    def call_args(self, args_w):
         # direct call: either pick non-templated overload or attempt to deduce
         # the template instantiation from the argument types
 
         # try existing overloads or compile-time instantiations
         try:
-            return W_CPPOverload.call(self, args_w)
+            return W_CPPOverload.call_args(self, args_w)
         except Exception:
             pass
 
@@ -814,6 +834,9 @@
     def getitem(self, args_w):
         return self.getitem_impl(self.name, args_w)
 
+    def getname(self, space):
+        return self.name
+
     def __repr__(self):
         return "W_CPPTemplateOverload(%s)" % [f.prototype() for f in self.functions]
 
@@ -821,7 +844,7 @@
     'CPPTemplateOverload',
     __get__     = interp2app(W_CPPTemplateOverload.descr_get),
     __getitem__ = interp2app(W_CPPTemplateOverload.getitem),
-    __call__    = interp2app(W_CPPTemplateOverload.call),
+    __call__    = interp2app(W_CPPTemplateOverload.call_args),
     __useffi__  = GetSetProperty(W_CPPTemplateOverload.fget_useffi, W_CPPTemplateOverload.fset_useffi),
     __doc__     = GetSetProperty(W_CPPTemplateOverload.fget_doc)
 )
@@ -830,18 +853,18 @@
     """App-level dispatcher to allow both lookup/instantiation of templated methods and
     dispatch among overloads between templated and non-templated method."""
 
-    _attrs_ = ['name', 'overloads', 'master']
+    _attrs_ = ['name', 'overloads', 'master', 'w_this']
     _immutable_fields_ = ['name']
 
-    def __init__(self, space, name, declaring_scope, functions, flags = OVERLOAD_FLAGS_USE_FFI):
-         W_CPPStaticOverload.__init__(self, space, declaring_scope, functions, flags)
+    def __init__(self, space, name, decl_scope, funcs, flags = OVERLOAD_FLAGS_USE_FFI):
+         W_CPPStaticOverload.__init__(self, space, decl_scope, funcs, flags)
          self.name = name
          self.overloads = {}
          self.master = self
+         self.w_this = space.w_None
 
-    @unwrap_spec(args_w='args_w')
-    def descr_get(self, w_cppinstance, args_w):
-        # like W_CPPStaticOverload, but returns W_CPPTemplateStaticOverload
+    def descr_get(self, w_cppinstance, w_cls=None):
+        # TODO: don't return copy, but bind in an external object (like W_CPPOverload)
         if isinstance(w_cppinstance, W_CPPInstance):
             cppinstance = self.space.interp_w(W_CPPInstance, w_cppinstance)
             if cppinstance.clsdecl.handle != self.scope.handle:
@@ -852,13 +875,13 @@
         return self      # unbound
 
     @unwrap_spec(args_w='args_w')
-    def call(self, args_w):
+    def call_args(self, args_w):
         # direct call: either pick non-templated overload or attempt to deduce
         # the template instantiation from the argument types
 
         # try existing overloads or compile-time instantiations
         try:
-            return W_CPPStaticOverload.call(self, args_w)
+            return W_CPPStaticOverload.call_args(self, args_w)
         except Exception:
             pass
 
@@ -869,6 +892,9 @@
     def getitem(self, args_w):
         return self.getitem_impl(self.name, args_w)
 
+    def getname(self, space):
+        return self.name
+
     def __repr__(self):
         return "W_CPPTemplateStaticOverload(%s)" % [f.prototype() for f in self.functions]
 
@@ -876,7 +902,7 @@
     'CPPTemplateStaticOverload',
     __get__     = interp2app(W_CPPTemplateStaticOverload.descr_get),
     __getitem__ = interp2app(W_CPPTemplateStaticOverload.getitem),
-    __call__    = interp2app(W_CPPTemplateStaticOverload.call),
+    __call__    = interp2app(W_CPPTemplateStaticOverload.call_args),
     __useffi__  = GetSetProperty(W_CPPTemplateStaticOverload.fget_useffi, W_CPPTemplateStaticOverload.fset_useffi),
     __doc__     = GetSetProperty(W_CPPTemplateStaticOverload.fget_doc)
 )
@@ -898,9 +924,9 @@
     _attrs_ = ['space', 'scope', 'converter', 'offset']
     _immutable_fields = ['scope', 'converter', 'offset']
 
-    def __init__(self, space, declaring_scope, type_name, offset):
+    def __init__(self, space, decl_scope, type_name, offset):
         self.space = space
-        self.scope = declaring_scope
+        self.scope = decl_scope
         self.converter = converter.get_converter(self.space, type_name, '')
         self.offset = offset
 
@@ -1417,7 +1443,7 @@
                     ol = W_CPPStaticOverload(self.space, nss, funcs[:])
                     # TODO: cache this operator (not done yet, as the above does not
                     # select all overloads)
-                    return ol.call([self, w_other])
+                    return ol.call_args([self, w_other])
         except OperationError as e:
             if not e.match(self.space, self.space.w_TypeError):
                 raise
diff --git a/pypy/module/_cppyy/test/test_zjit.py b/pypy/module/_cppyy/test/test_zjit.py
--- a/pypy/module/_cppyy/test/test_zjit.py
+++ b/pypy/module/_cppyy/test/test_zjit.py
@@ -205,6 +205,9 @@
     def exception_match(self, typ, sub):
         return typ is sub
 
+    def is_none(self, w_obj):
+        return w_obj is None
+
     def is_w(self, w_one, w_two):
         return w_one is w_two
 
@@ -268,6 +271,9 @@
     def call_function(self, w_func, *args_w):
         return None
 
+    def call_obj_args(self, w_callable, w_obj, args):
+        return w_callable.call_args([w_obj]+args)
+
     def _freeze_(self):
         return True
 
@@ -283,19 +289,19 @@
         def f():
             cls  = interp_cppyy.scope_byname(space, "example01")
             inst = interp_cppyy._bind_object(space, FakeInt(0), cls, True)
-            cls.get_overload("__init__").descr_get(inst, []).call([FakeInt(0)])
+            cls.get_overload("__init__").descr_get(inst, []).call_args([FakeInt(0)])
             cppmethod = cls.get_overload(method_name)
             assert isinstance(inst, interp_cppyy.W_CPPInstance)
             i = 10
             while i > 0:
                 drv.jit_merge_point(inst=inst, cppmethod=cppmethod, i=i)
-                cppmethod.descr_get(inst, []).call([FakeInt(i)])
+                cppmethod.descr_get(inst, []).call_args([FakeInt(i)])
                 i -= 1
             return 7
         f()
         space = FakeSpace()
         result = self.meta_interp(f, [], listops=True, backendopt=True, listcomp=True)
-        self.check_jitcell_token_count(0)   # same for fast and slow path??
+        self.check_jitcell_token_count(1)
         # rely on replacement of capi calls to raise exception instead (see FakeSpace.__init__)
 
     @py.test.mark.dont_track_allocations("cppmethod.cif_descr kept 'leaks'")
diff --git a/pypy/module/cpyext/cdatetime.py b/pypy/module/cpyext/cdatetime.py
--- a/pypy/module/cpyext/cdatetime.py
+++ b/pypy/module/cpyext/cdatetime.py
@@ -120,21 +120,10 @@
                    dealloc=type_dealloc,
                   )
 
-    # why do we need date_dealloc? Since W_DateTime_Date is the base class for
-    # app level datetime.date. If a c-extension class uses datetime.date for its
-    # base class and defines a tp_dealloc, we will get this:
-    # c_class->tp_dealloc == tp_dealloc_func
-    # c_class->tp_base == datetime.date, 
-    #                     datetime.date->tp_dealloc = _PyPy_subtype_dealloc
-    # datetime.date->tp_base = W_DateTime_Date
-    #                    W_DateTime_Date->tp_dealloc = _PyPy_subtype_dealloc
-    # but _PyPy_subtype_dealloc will call tp_dealloc_func, which can call its
-    # base's tp_dealloc and we get recursion. So break the recursion by setting
-    # W_DateTime_Date->tp_dealloc
     make_typedescr(W_DateTime_Date.typedef,
                    basestruct=PyDateTime_DateTime.TO,
                    attach=type_attach,
-                   dealloc=date_dealloc,
+                   dealloc=type_dealloc,
                   )
 
     make_typedescr(W_DateTime_Delta.typedef,
@@ -144,30 +133,41 @@
 
 def type_attach(space, py_obj, w_obj, w_userdata=None):
     '''Fills a newly allocated py_obj from the w_obj
+    If it is a datetime.time or datetime.datetime, it may have tzinfo
     '''
-    if space.type(w_obj).name == 'date':
-        # No tzinfo
-        return
-    py_datetime = rffi.cast(PyDateTime_Time, py_obj)
-    w_tzinfo = space.getattr(w_obj, space.newtext('tzinfo'))
-    if space.is_none(w_tzinfo):
-        py_datetime.c_hastzinfo = cts.cast('unsigned char', 0)
-        py_datetime.c_tzinfo = lltype.nullptr(PyObject.TO)
-    else:
-        py_datetime.c_hastzinfo = cts.cast('unsigned char', 1)
-        py_datetime.c_tzinfo = make_ref(space, w_tzinfo)
+    assert len(datetimeAPI_global) > 0
+    if datetimeAPI_global[0].c_TimeType == py_obj.c_ob_type:
+        py_datetime = rffi.cast(PyDateTime_Time, py_obj)
+        w_tzinfo = space.getattr(w_obj, space.newtext('tzinfo'))
+        if space.is_none(w_tzinfo):
+            py_datetime.c_hastzinfo = cts.cast('unsigned char', 0)
+            py_datetime.c_tzinfo = lltype.nullptr(PyObject.TO)
+        else:
+            py_datetime.c_hastzinfo = cts.cast('unsigned char', 1)
+            py_datetime.c_tzinfo = make_ref(space, w_tzinfo)
+    elif datetimeAPI_global[0].c_DateTimeType == py_obj.c_ob_type:
+        # For now this is exactly the same structure as PyDateTime_Time
+        py_datetime = rffi.cast(PyDateTime_DateTime, py_obj)
+        w_tzinfo = space.getattr(w_obj, space.newtext('tzinfo'))
+        if space.is_none(w_tzinfo):
+            py_datetime.c_hastzinfo = cts.cast('unsigned char', 0)
+            py_datetime.c_tzinfo = lltype.nullptr(PyObject.TO)
+        else:
+            py_datetime.c_hastzinfo = cts.cast('unsigned char', 1)
+            py_datetime.c_tzinfo = make_ref(space, w_tzinfo)
 
 @slot_function([PyObject], lltype.Void)
 def type_dealloc(space, py_obj):
-    py_datetime = rffi.cast(PyDateTime_Time, py_obj)
-    if (widen(py_datetime.c_hastzinfo) != 0):
-        decref(space, py_datetime.c_tzinfo)
     from pypy.module.cpyext.object import _dealloc
-    _dealloc(space, py_obj)
-
- at slot_function([PyObject], lltype.Void)
-def date_dealloc(space, py_obj):
-    from pypy.module.cpyext.object import _dealloc
+    assert len(datetimeAPI_global) > 0
+    if datetimeAPI_global[0].c_TimeType == py_obj.c_ob_type:
+        py_datetime = rffi.cast(PyDateTime_Time, py_obj)
+        if (widen(py_datetime.c_hastzinfo) != 0):
+            decref(space, py_datetime.c_tzinfo)
+    elif datetimeAPI_global[0].c_DateTimeType == py_obj.c_ob_type:
+        py_datetime = rffi.cast(PyDateTime_DateTime, py_obj)
+        if (widen(py_datetime.c_hastzinfo) != 0):
+            decref(space, py_datetime.c_tzinfo)
     _dealloc(space, py_obj)
 
 def timedeltatype_attach(space, py_obj, w_obj, w_userdata=None):
diff --git a/pypy/module/cpyext/test/test_datetime.py b/pypy/module/cpyext/test/test_datetime.py
--- a/pypy/module/cpyext/test/test_datetime.py
+++ b/pypy/module/cpyext/test/test_datetime.py
@@ -307,18 +307,20 @@
         from datetime import tzinfo, datetime, timedelta, time
         # copied from datetime documentation
         class GMT1(tzinfo):
-           def utcoffset(self, dt):
-               return timedelta(hours=1) + self.dst(dt)
-           def dst(self, dt):
-               return timedelta(0)
-           def tzname(self,dt):
+            def __del__(self):
+                print 'deleting GMT1'
+            def utcoffset(self, dt):
+                return timedelta(hours=1) + self.dst(dt)
+            def dst(self, dt):
+                return timedelta(0)
+            def tzname(self,dt):
                 return "GMT +1"
         gmt1 = GMT1()
         dt1 = module.time_with_tzinfo(gmt1)
         assert dt1 == time(6, 6, 6, 6, gmt1)
         assert '+01' in str(dt1)
-        assert module.datetime_with_tzinfo(gmt1) == datetime(
-            2000, 6, 6, 6, 6, 6, 6, gmt1)
+        dt_tz = module.datetime_with_tzinfo(gmt1)
+        assert dt_tz == datetime(2000, 6, 6, 6, 6, 6, 6, gmt1)
 
     def test_checks(self):
         module = self.import_extension('foo', [
diff --git a/pypy/module/sys/app.py b/pypy/module/sys/app.py
--- a/pypy/module/sys/app.py
+++ b/pypy/module/sys/app.py
@@ -11,6 +11,11 @@
 
     # Flush stdout as well, both files may refer to the same file
     try:
+        if sys.stdout.softspace:
+            print
+    except:
+        pass
+    try:
         sys.stdout.flush()
     except:
         pass
diff --git a/pypy/module/sys/test/test_sysmodule.py b/pypy/module/sys/test/test_sysmodule.py
--- a/pypy/module/sys/test/test_sysmodule.py
+++ b/pypy/module/sys/test/test_sysmodule.py
@@ -256,6 +256,30 @@
             print repr(err.getvalue())
             assert err.getvalue().endswith("ValueError: %s\n" % expectedoutput)
 
+    def test_excepthook_flushes_stdout(self):
+        import sys, cStringIO
+        savestdout = sys.stdout
+        out = cStringIO.StringIO()
+        sys.stdout = out
+
+        eh = sys.__excepthook__
+
+        try:
+            raise ValueError(42)
+        except ValueError as exc:
+            print "hello"     # with end-of-line
+            eh(*sys.exc_info())
+        try:
+            raise ValueError(42)
+        except ValueError as exc:
+            print 123, 456,     # no end-of-line here
+            assert sys.stdout.softspace
+            eh(*sys.exc_info())
+            assert not sys.stdout.softspace
+
+        sys.stdout = savestdout
+        assert out.getvalue() == 'hello\n123 456\n'   # with a final \n added
+
     # FIXME: testing the code for a lost or replaced excepthook in
     # Python/pythonrun.c::PyErr_PrintEx() is tricky.
 
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py b/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py
@@ -1824,19 +1824,39 @@
         ffi = FFI(backend=self.Backend())
         ffi.cdef("struct nonpacked { char a; int b; };")
         ffi.cdef("struct is_packed { char a; int b; };", packed=True)
+        ffi.cdef("struct is_packed1 { char a; int b; };", pack=1)
+        ffi.cdef("struct is_packed2 { char a; int b; };", pack=2)
+        ffi.cdef("struct is_packed4 { char a; int b; };", pack=4)
+        ffi.cdef("struct is_packed8 { char a; int b; };", pack=8)
         assert ffi.sizeof("struct nonpacked") == 8
         assert ffi.sizeof("struct is_packed") == 5
+        assert ffi.sizeof("struct is_packed1") == 5
+        assert ffi.sizeof("struct is_packed2") == 6
+        assert ffi.sizeof("struct is_packed4") == 8
+        assert ffi.sizeof("struct is_packed8") == 8
         assert ffi.alignof("struct nonpacked") == 4
         assert ffi.alignof("struct is_packed") == 1
-        s = ffi.new("struct is_packed[2]")
-        s[0].b = 42623381
-        s[0].a = b'X'
-        s[1].b = -4892220
-        s[1].a = b'Y'
-        assert s[0].b == 42623381
-        assert s[0].a == b'X'
-        assert s[1].b == -4892220
-        assert s[1].a == b'Y'
+        assert ffi.alignof("struct is_packed1") == 1
+        assert ffi.alignof("struct is_packed2") == 2
+        assert ffi.alignof("struct is_packed4") == 4
+        assert ffi.alignof("struct is_packed8") == 4
+        for name in ['is_packed', 'is_packed1', 'is_packed2',
+                     'is_packed4', 'is_packed8']:
+            s = ffi.new("struct %s[2]" % name)
+            s[0].b = 42623381
+            s[0].a = b'X'
+            s[1].b = -4892220
+            s[1].a = b'Y'
+            assert s[0].b == 42623381
+            assert s[0].a == b'X'
+            assert s[1].b == -4892220
+            assert s[1].a == b'Y'
+
+    def test_pack_valueerror(self):
+        ffi = FFI(backend=self.Backend())
+        py.test.raises(ValueError, ffi.cdef, "", pack=3)
+        py.test.raises(ValueError, ffi.cdef, "", packed=2)
+        py.test.raises(ValueError, ffi.cdef, "", packed=True, pack=1)
 
     def test_define_integer_constant(self):
         ffi = FFI(backend=self.Backend())
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
@@ -2243,6 +2243,12 @@
         "'API mode' and non-variadic (i.e. declared inside ffibuilder.cdef()"
         "+ffibuilder.set_source() and not taking a final '...' argument)")
 
+def test_pack_not_supported():
+    ffi = FFI()
+    ffi.cdef("""struct foo { char y; int x; };""", pack=2)
+    py.test.raises(NotImplementedError, verify,
+                   ffi, "test_pack_not_supported", "")
+
 def test_gcc_visibility_hidden():
     if sys.platform == 'win32':
         py.test.skip("test for gcc/clang")
diff --git a/pypy/module/test_lib_pypy/test_code_extra.py b/pypy/module/test_lib_pypy/test_code_extra.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/test_lib_pypy/test_code_extra.py
@@ -0,0 +1,19 @@
+import py
+import sys
+import cStringIO
+import code
+
+
+def test_flush_stdout_on_error():
+    runner = code.InteractiveInterpreter()
+    old_stdout = sys.stdout
+    try:
+        mystdout = cStringIO.StringIO()
+        sys.stdout = mystdout
+        runner.runcode(compile("print 5,;0/0", "<interactive>", "exec"))
+    finally:
+        sys.stdout = old_stdout
+
+    if '__pypy__' not in sys.builtin_module_names:
+        py.test.skip('pypy only test')
+    assert mystdout.getvalue() == "5\n"
diff --git a/pypy/objspace/std/intobject.py b/pypy/objspace/std/intobject.py
--- a/pypy/objspace/std/intobject.py
+++ b/pypy/objspace/std/intobject.py
@@ -233,20 +233,23 @@
     return wrapint(space, a)
 
 
- at jit.look_inside_iff(lambda space, iv, iw, iz:
-                     jit.isconstant(iw) and jit.isconstant(iz))
 def _pow(space, iv, iw, iz):
     """Helper for pow"""
-    if iw < 0:
-        if iz != 0:
-            raise oefmt(space.w_TypeError,
-                        "pow() 2nd argument cannot be negative when 3rd "
-                        "argument specified")
+    if iz == 0:
+        return _pow_nomod(iv, iw)
+    else:
+        return _pow_mod(space, iv, iw, iz)
+
+ at jit.look_inside_iff(lambda iv, iw: jit.isconstant(iw))
+def _pow_nomod(iv, iw):
+    if iw <= 0:
+        if iw == 0:
+            return 1
         # bounce it, since it always returns float
         raise ValueError
     temp = iv
     ix = 1
-    while iw > 0:
+    while True:
         if iw & 1:
             try:
                 ix = ovfcheck(ix * temp)
@@ -259,12 +262,40 @@
             temp = ovfcheck(temp * temp) # Square the value of temp
         except OverflowError:
             raise
-        if iz:
-            # If we did a multiplication, perform a modulo
-            ix %= iz
-            temp %= iz
-    if iz:
-        ix %= iz
+    return ix
+
+ at jit.look_inside_iff(lambda space, iv, iw, iz:
+                     jit.isconstant(iw) and jit.isconstant(iz))
+def _pow_mod(space, iv, iw, iz):
+    from rpython.rlib.rarithmetic import mulmod
+
+    if iw <= 0:
+        if iw == 0:
+            return 1 % iz   # != 1, for iz == 1 or iz < 0
+        raise oefmt(space.w_TypeError,
+                    "pow() 2nd argument cannot be negative when 3rd "
+                    "argument specified")
+    if iz < 0:
+        try:
+            iz = ovfcheck(-iz)
+        except OverflowError:
+            raise
+        iz_negative = True
+    else:
+        iz_negative = False
+
+    temp = iv
+    ix = 1
+    while True:
+        if iw & 1:
+            ix = mulmod(ix, temp, iz)
+        iw >>= 1   # Shift exponent down by 1 bit
+        if iw == 0:
+            break
+        temp = mulmod(temp, temp, iz)
+
+    if iz_negative and ix > 0:
+        ix -= iz
     return ix
 
 
diff --git a/pypy/objspace/std/test/test_intobject.py b/pypy/objspace/std/test/test_intobject.py
--- a/pypy/objspace/std/test/test_intobject.py
+++ b/pypy/objspace/std/test/test_intobject.py
@@ -1,8 +1,9 @@
 # encoding: utf-8
 import py
 import sys
+from pypy.interpreter.error import OperationError
 from pypy.objspace.std import intobject as iobj
-from rpython.rlib.rarithmetic import r_uint, is_valid_int
+from rpython.rlib.rarithmetic import r_uint, is_valid_int, intmask
 from rpython.rlib.rbigint import rbigint
 
 
@@ -186,6 +187,63 @@
         assert space.isinstance_w(v, space.w_long)
         assert space.bigint_w(v).eq(rbigint.fromlong(pow(10, 20)))
 
+    try:
+        from hypothesis import given, strategies, example
+    except ImportError:
+        pass
+    else:
+        @given(
+           a=strategies.integers(min_value=-sys.maxint-1, max_value=sys.maxint),
+           b=strategies.integers(min_value=-sys.maxint-1, max_value=sys.maxint),
+           c=strategies.integers(min_value=-sys.maxint-1, max_value=sys.maxint))
+        @example(0, 0, -sys.maxint-1)
+        @example(0, 1, -sys.maxint-1)
+        @example(1, 0, -sys.maxint-1)
+        def test_hypot_pow(self, a, b, c):
+            if c == 0:
+                return
+            #
+            # "pow(a, b, c)": if b < 0, should get an app-level TypeError.
+            # Otherwise, should always work except if c == -maxint-1
+            if b < 0:
+                expected = TypeError
+            elif b > 0 and c == -sys.maxint-1:
+                expected = OverflowError
+            else:
+                expected = pow(a, b, c)
+
+            try:
+                result = iobj._pow(self.space, a, b, c)
+            except OperationError as e:
+                assert ('TypeError: pow() 2nd argument cannot be negative '
+                        'when 3rd argument specified' == e.errorstr(self.space))
+                result = TypeError
+            except OverflowError:
+                result = OverflowError
+            assert result == expected
+
+        @given(
+           a=strategies.integers(min_value=-sys.maxint-1, max_value=sys.maxint),
+           b=strategies.integers(min_value=-sys.maxint-1, max_value=sys.maxint))
+        def test_hypot_pow_nomod(self, a, b):
+            # "a ** b": detect overflows and ValueErrors
+            if b < 0:
+                expected = ValueError
+            elif b > 128 and not (-1 <= a <= 1):
+                expected = OverflowError
+            else:
+                expected = a ** b
+                if expected != intmask(expected):
+                    expected = OverflowError
+
+            try:
+                result = iobj._pow(self.space, a, b, 0)
+            except ValueError:
+                result = ValueError
+            except OverflowError:
+                result = OverflowError
+            assert result == expected
+
     def test_neg(self):
         space = self.space
         x = 42
diff --git a/rpython/rlib/rarithmetic.py b/rpython/rlib/rarithmetic.py
--- a/rpython/rlib/rarithmetic.py
+++ b/rpython/rlib/rarithmetic.py
@@ -840,6 +840,31 @@
         return z
 
 
+ at specialize.memo()
+def check_support_int128():
+    from rpython.rtyper.lltypesystem import rffi
+    return hasattr(rffi, '__INT128_T')
+
+def mulmod(a, b, c):
+    """Computes (a * b) % c.
+    Assumes c > 0, and returns a nonnegative result.
+    """
+    assert c > 0
+    if LONG_BIT < LONGLONG_BIT:
+        a = r_longlong(a)
+        b = r_longlong(b)
+        return intmask((a * b) % c)
+    elif check_support_int128():
+        a = r_longlonglong(a)
+        b = r_longlonglong(b)
+        return intmask((a * b) % c)
+    else:
+        from rpython.rlib.rbigint import rbigint
+        a = rbigint.fromlong(a)
+        b = rbigint.fromlong(b)
+        return a.mul(b).int_mod(c).toint()
+
+
 # String parsing support
 # ---------------------------
 
diff --git a/rpython/rlib/rbigint.py b/rpython/rlib/rbigint.py
--- a/rpython/rlib/rbigint.py
+++ b/rpython/rlib/rbigint.py
@@ -1,6 +1,7 @@
 from rpython.rlib.rarithmetic import LONG_BIT, intmask, longlongmask, r_uint, r_ulonglong
 from rpython.rlib.rarithmetic import ovfcheck, r_longlong, widen
 from rpython.rlib.rarithmetic import most_neg_value_of_same_type
+from rpython.rlib.rarithmetic import check_support_int128

From pypy.commits at gmail.com  Fri Jul 27 16:02:17 2018
From: pypy.commits at gmail.com (mattip)
Date: Fri, 27 Jul 2018 13:02:17 -0700 (PDT)
Subject: [pypy-commit] pypy unicode-utf8-py3: merge py3.5 into branch
Message-ID: <5b5b7a49.1c69fb81.d9631.7c34@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: unicode-utf8-py3
Changeset: r94901:d50076cea2ec
Date: 2018-07-11 09:26 -0500
http://bitbucket.org/pypy/pypy/changeset/d50076cea2ec/

Log:	merge py3.5 into branch

diff --git a/lib_pypy/cffi.egg-info/PKG-INFO b/lib_pypy/cffi.egg-info/PKG-INFO
--- a/lib_pypy/cffi.egg-info/PKG-INFO
+++ b/lib_pypy/cffi.egg-info/PKG-INFO
@@ -1,6 +1,6 @@
 Metadata-Version: 1.1
 Name: cffi
-Version: 1.11.5
+Version: 1.12.0
 Summary: Foreign Function Interface for Python calling C code.
 Home-page: http://cffi.readthedocs.org
 Author: Armin Rigo, Maciej Fijalkowski
diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py
--- a/lib_pypy/cffi/__init__.py
+++ b/lib_pypy/cffi/__init__.py
@@ -4,8 +4,8 @@
 from .api import FFI
 from .error import CDefError, FFIError, VerificationError, VerificationMissing
 
-__version__ = "1.11.5"
-__version_info__ = (1, 11, 5)
+__version__ = "1.12.0"
+__version_info__ = (1, 12, 0)
 
 # The verifier module file names are based on the CRC32 of a string that
 # contains the following version number.  It may be older than __version__
diff --git a/lib_pypy/cffi/_embedding.h b/lib_pypy/cffi/_embedding.h
--- a/lib_pypy/cffi/_embedding.h
+++ b/lib_pypy/cffi/_embedding.h
@@ -221,7 +221,7 @@
 
         if (f != NULL && f != Py_None) {
             PyFile_WriteString("\nFrom: " _CFFI_MODULE_NAME
-                               "\ncompiled with cffi version: 1.11.5"
+                               "\ncompiled with cffi version: 1.12.0"
                                "\n_cffi_backend module: ", f);
             modules = PyImport_GetModuleDict();
             mod = PyDict_GetItemString(modules, "_cffi_backend");
diff --git a/lib_pypy/cffi/api.py b/lib_pypy/cffi/api.py
--- a/lib_pypy/cffi/api.py
+++ b/lib_pypy/cffi/api.py
@@ -96,18 +96,21 @@
             self.CData, self.CType = backend._get_types()
         self.buffer = backend.buffer
 
-    def cdef(self, csource, override=False, packed=False):
+    def cdef(self, csource, override=False, packed=False, pack=None):
         """Parse the given C source.  This registers all declared functions,
         types, and global variables.  The functions and global variables can
         then be accessed via either 'ffi.dlopen()' or 'ffi.verify()'.
         The types can be used in 'ffi.new()' and other functions.
         If 'packed' is specified as True, all structs declared inside this
         cdef are packed, i.e. laid out without any field alignment at all.
+        Alternatively, 'pack' can be a small integer, and requests for
+        alignment greater than that are ignored (pack=1 is equivalent to
+        packed=True).
         """
-        self._cdef(csource, override=override, packed=packed)
+        self._cdef(csource, override=override, packed=packed, pack=pack)
 
-    def embedding_api(self, csource, packed=False):
-        self._cdef(csource, packed=packed, dllexport=True)
+    def embedding_api(self, csource, packed=False, pack=None):
+        self._cdef(csource, packed=packed, pack=pack, dllexport=True)
         if self._embedding is None:
             self._embedding = ''
 
diff --git a/lib_pypy/cffi/backend_ctypes.py b/lib_pypy/cffi/backend_ctypes.py
--- a/lib_pypy/cffi/backend_ctypes.py
+++ b/lib_pypy/cffi/backend_ctypes.py
@@ -730,7 +730,8 @@
         return self._new_struct_or_union('union', name, ctypes.Union)
 
     def complete_struct_or_union(self, CTypesStructOrUnion, fields, tp,
-                                 totalsize=-1, totalalignment=-1, sflags=0):
+                                 totalsize=-1, totalalignment=-1, sflags=0,
+                                 pack=0):
         if totalsize >= 0 or totalalignment >= 0:
             raise NotImplementedError("the ctypes backend of CFFI does not support "
                                       "structures completed by verify(); please "
@@ -751,6 +752,8 @@
                 bfield_types[fname] = Ellipsis
         if sflags & 8:
             struct_or_union._pack_ = 1
+        elif pack:
+            struct_or_union._pack_ = pack
         struct_or_union._fields_ = cfields
         CTypesStructOrUnion._bfield_types = bfield_types
         #
diff --git a/lib_pypy/cffi/cparser.py b/lib_pypy/cffi/cparser.py
--- a/lib_pypy/cffi/cparser.py
+++ b/lib_pypy/cffi/cparser.py
@@ -306,11 +306,25 @@
             msg = 'parse error\n%s' % (msg,)
         raise CDefError(msg)
 
-    def parse(self, csource, override=False, packed=False, dllexport=False):
+    def parse(self, csource, override=False, packed=False, pack=None,
+                    dllexport=False):
+        if packed:
+            if packed != True:
+                raise ValueError("'packed' should be False or True; use "
+                                 "'pack' to give another value")
+            if pack:
+                raise ValueError("cannot give both 'pack' and 'packed'")
+            pack = 1
+        elif pack:
+            if pack & (pack - 1):
+                raise ValueError("'pack' must be a power of two, not %r" %
+                    (pack,))
+        else:
+            pack = 0
         prev_options = self._options
         try:
             self._options = {'override': override,
-                             'packed': packed,
+                             'packed': pack,
                              'dllexport': dllexport}
             self._internal_parse(csource)
         finally:
diff --git a/lib_pypy/cffi/model.py b/lib_pypy/cffi/model.py
--- a/lib_pypy/cffi/model.py
+++ b/lib_pypy/cffi/model.py
@@ -342,7 +342,7 @@
     fixedlayout = None
     completed = 0
     partial = False
-    packed = False
+    packed = 0
 
     def __init__(self, name, fldnames, fldtypes, fldbitsize, fldquals=None):
         self.name = name
@@ -414,11 +414,14 @@
             fldtypes = [tp.get_cached_btype(ffi, finishlist)
                         for tp in self.fldtypes]
             lst = list(zip(self.fldnames, fldtypes, self.fldbitsize))
-            sflags = 0
+            extra_flags = ()
             if self.packed:
-                sflags = 8    # SF_PACKED
+                if self.packed == 1:
+                    extra_flags = (8,)    # SF_PACKED
+                else:
+                    extra_flags = (0, self.packed)
             ffi._backend.complete_struct_or_union(BType, lst, self,
-                                                  -1, -1, sflags)
+                                                  -1, -1, *extra_flags)
             #
         else:
             fldtypes = []
diff --git a/lib_pypy/cffi/recompiler.py b/lib_pypy/cffi/recompiler.py
--- a/lib_pypy/cffi/recompiler.py
+++ b/lib_pypy/cffi/recompiler.py
@@ -893,6 +893,12 @@
             else:
                 flags.append("_CFFI_F_CHECK_FIELDS")
             if tp.packed:
+                if tp.packed > 1:
+                    raise NotImplementedError(
+                        "%r is declared with 'pack=%r'; only 0 or 1 are "
+                        "supported in API mode (try to use \"...;\", which "
+                        "does not require a 'pack' declaration)" %
+                        (tp, tp.packed))
                 flags.append("_CFFI_F_PACKED")
         else:
             flags.append("_CFFI_F_EXTERNAL")
diff --git a/pypy/doc/config/objspace.disable_entrypoints.txt b/pypy/doc/config/objspace.disable_entrypoints.txt
new file mode 100644
diff --git a/pypy/doc/config/objspace.fstrings.txt b/pypy/doc/config/objspace.fstrings.txt
new file mode 100644
diff --git a/pypy/doc/config/objspace.hash.txt b/pypy/doc/config/objspace.hash.txt
new file mode 100644
diff --git a/pypy/doc/config/objspace.usemodules._frozen_importlib.txt b/pypy/doc/config/objspace.usemodules._frozen_importlib.txt
new file mode 100644
diff --git a/pypy/doc/config/objspace.usemodules._jitlog.txt b/pypy/doc/config/objspace.usemodules._jitlog.txt
new file mode 100644
diff --git a/pypy/doc/config/objspace.usemodules.faulthandler.txt b/pypy/doc/config/objspace.usemodules.faulthandler.txt
new file mode 100644
diff --git a/pypy/doc/config/translation.backendopt.replace_we_are_jitted.txt b/pypy/doc/config/translation.backendopt.replace_we_are_jitted.txt
new file mode 100644
diff --git a/pypy/doc/config/translation.jit_opencoder_model.txt b/pypy/doc/config/translation.jit_opencoder_model.txt
new file mode 100644
diff --git a/pypy/doc/config/translation.keepgoing.txt b/pypy/doc/config/translation.keepgoing.txt
new file mode 100644
diff --git a/pypy/doc/config/translation.libname.txt b/pypy/doc/config/translation.libname.txt
new file mode 100644
diff --git a/pypy/doc/config/translation.lto.txt b/pypy/doc/config/translation.lto.txt
new file mode 100644
diff --git a/pypy/doc/config/translation.profoptargs.txt b/pypy/doc/config/translation.profoptargs.txt
new file mode 100644
diff --git a/pypy/doc/config/translation.reverse_debugger.txt b/pypy/doc/config/translation.reverse_debugger.txt
new file mode 100644
diff --git a/pypy/doc/config/translation.split_gc_address_space.txt b/pypy/doc/config/translation.split_gc_address_space.txt
new file mode 100644
diff --git a/pypy/doc/tool/makecontributor.py b/pypy/doc/tool/makecontributor.py
--- a/pypy/doc/tool/makecontributor.py
+++ b/pypy/doc/tool/makecontributor.py
@@ -18,12 +18,13 @@
     'Antonio Cuni': ['antocuni', 'anto'],
     'Armin Rigo': ['arigo', 'arfigo', 'armin', 'arigato'],
     'Maciej Fijalkowski': ['fijal'],
-    'Carl Friedrich Bolz-Tereick': ['Carl Friedrich Bolz', 'cfbolz', 'cf'],
+    'Carl Friedrich Bolz-Tereick': ['Carl Friedrich Bolz', 'cfbolz', 'cf', 'cbolz'],
     'Samuele Pedroni': ['pedronis', 'samuele', 'samule'],
-    'Richard Plangger':['planrich'],
-    'Michael Hudson': ['mwh'],
+    'Richard Plangger': ['planrich', 'plan_rich'],
+    'Remi Meier': ['remi'],
+    'Michael Hudson-Doyle': ['mwh', 'Michael Hudson'],
     'Holger Krekel': ['hpk', 'holger krekel', 'holger', 'hufpk'],
-    "Amaury Forgeot d'Arc": ['afa'],
+    "Amaury Forgeot d'Arc": ['afa', 'amauryfa at gmail.com'],
     'Alex Gaynor': ['alex', 'agaynor'],
     'David Schneider': ['bivab', 'david'],
     'Christian Tismer': ['chris', 'christian', 'tismer',
@@ -41,7 +42,7 @@
     'Mark Pearse': ['mwp'],
     'Toon Verwaest': ['tverwaes'],
     'Eric van Riet Paap': ['ericvrp'],
-    'Jacob Hallen': ['jacob', 'jakob'],
+    'Jacob Hallen': ['jacob', 'jakob', 'jacob hallen'],
     'Anders Lehmann': ['ale', 'anders'],
     'Bert Freudenberg': ['bert'],
     'Boris Feigin': ['boris', 'boria'],
@@ -69,19 +70,25 @@
     'Manuel Jacob': ['mjacob'],
     'Rami Chowdhury': ['necaris'],
     'Stanislaw Halik': ['Stanislaw Halik', 'w31rd0'],
-    'Wenzhu Man':['wenzhu man', 'wenzhuman'],
-    'Anton Gulenko':['anton gulenko', 'anton_gulenko'],
-    'Richard Lancaster':['richardlancaster'],
-    'William Leslie':['William ML Leslie'],
-    'Spenser Bauman':['Spenser Andrew Bauman'],
-    'Raffael Tfirst':['raffael.tfirst at gmail.com'],
-    'timo':['timo at eistee.fritz.box'],
-    'Jasper Schulz':['Jasper.Schulz', 'jbs'],
-    'Aaron Gallagher':['"Aaron Gallagher'],
-    'Yasir Suhail':['yasirs'],
+    'Wenzhu Man': ['wenzhu man', 'wenzhuman'],
+    'Anton Gulenko': ['anton gulenko', 'anton_gulenko'],
+    'Richard Lancaster': ['richardlancaster'],
+    'William Leslie': ['William ML Leslie'],
+    'Spenser Bauman': ['Spenser Andrew Bauman'],
+    'Raffael Tfirst': ['raffael.tfirst at gmail.com'],
+    'timo': ['timo at eistee.fritz.box'],
+    'Jasper Schulz': ['Jasper.Schulz', 'jbs'],
+    'Aaron Gallagher': ['"Aaron Gallagher'],
+    'Yasir Suhail': ['yasirs'],
     'Squeaky': ['squeaky'],
-    "Amaury Forgeot d'Arc": ['amauryfa at gmail.com'],
     "Dodan Mihai": ['mihai.dodan at gmail.com'],
+    'Wim Lavrijsen': ['wlav'],
+    'Toon Verwaest': ['toon', 'tverwaes'],
+    'Seo Sanghyeon': ['sanxiyn'],
+    'Leonardo Santagada': ['santagada'],
+    'Laurence Tratt': ['ltratt'],
+    'Pieter Zieschang': ['pzieschang', 'p_zieschang at yahoo.de'],
+    'John Witulski': ['witulski'],
     }
 
 alias_map = {}
@@ -103,7 +110,8 @@
         return set()
     ignore_words = ['around', 'consulting', 'yesterday', 'for a bit', 'thanks',
                     'in-progress', 'bits of', 'even a little', 'floating',
-                    'a bit', 'reviewing']
+                    'a bit', 'reviewing', 'looking', 'advising', 'partly', 'ish',
+                    'watching', 'mostly', 'jumping']
     sep_words = ['and', ';', '+', '/', 'with special  by']
     nicknames = match.group(1)
     for word in ignore_words:
diff --git a/pypy/module/_cffi_backend/__init__.py b/pypy/module/_cffi_backend/__init__.py
--- a/pypy/module/_cffi_backend/__init__.py
+++ b/pypy/module/_cffi_backend/__init__.py
@@ -3,7 +3,7 @@
 from rpython.rlib import rdynload, clibffi
 from rpython.rtyper.lltypesystem import rffi
 
-VERSION = "1.11.5"
+VERSION = "1.12.0"
 
 FFI_DEFAULT_ABI = clibffi.FFI_DEFAULT_ABI
 try:
diff --git a/pypy/module/_cffi_backend/newtype.py b/pypy/module/_cffi_backend/newtype.py
--- a/pypy/module/_cffi_backend/newtype.py
+++ b/pypy/module/_cffi_backend/newtype.py
@@ -258,6 +258,11 @@
 SF_PACKED             = 0x08
 SF_STD_FIELD_POS      = 0x80
 
+if sys.platform == 'win32':
+    SF_DEFAULT_PACKING = 8
+else:
+    SF_DEFAULT_PACKING = 0x40000000    # a huge power of two
+
 
 if sys.platform == 'win32':
     DEFAULT_SFLAGS_PLATFORM = SF_MSVC_BITFIELDS
@@ -309,10 +314,18 @@
         w_ctype._custom_field_pos = True
 
 @unwrap_spec(w_ctype=ctypeobj.W_CType, totalsize=int, totalalignment=int,
-             sflags=int)
+             sflags=int, pack=int)
 def complete_struct_or_union(space, w_ctype, w_fields, w_ignored=None,
-                             totalsize=-1, totalalignment=-1, sflags=0):
+                             totalsize=-1, totalalignment=-1, sflags=0,
+                             pack=0):
     sflags = complete_sflags(sflags)
+    if sflags & SF_PACKED:
+        pack = 1
+    elif pack <= 0:
+        pack = SF_DEFAULT_PACKING
+    else:
+        sflags |= SF_PACKED
+
     if (not isinstance(w_ctype, ctypestruct.W_CTypeStructOrUnion)
             or w_ctype.size >= 0):
         raise oefmt(space.w_TypeError,
@@ -362,7 +375,7 @@
         # update the total alignment requirement, but skip it if the
         # field is an anonymous bitfield or if SF_PACKED
         falignorg = ftype.alignof()
-        falign = 1 if sflags & SF_PACKED else falignorg
+        falign = min(pack, falignorg)
         do_align = True
         if (sflags & SF_GCC_ARM_BITFIELDS) == 0 and fbitsize >= 0:
             if (sflags & SF_MSVC_BITFIELDS) == 0:
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -1,7 +1,7 @@
 # ____________________________________________________________
 
 import sys
-assert __version__ == "1.11.5", ("This test_c.py file is for testing a version"
+assert __version__ == "1.12.0", ("This test_c.py file is for testing a version"
                                  " of cffi that differs from the one that we"
                                  " get from 'import _cffi_backend'")
 if sys.version_info < (3,):
@@ -3541,30 +3541,50 @@
     BLong = new_primitive_type("long")
     BChar = new_primitive_type("char")
     BShort = new_primitive_type("short")
-    BStruct = new_struct_type("struct foo")
-    complete_struct_or_union(BStruct, [('a1', BLong, -1),
-                                       ('a2', BChar, -1),
-                                       ('a3', BShort, -1)],
-                             None, -1, -1, SF_PACKED)
-    d = BStruct.fields
-    assert len(d) == 3
-    assert d[0][0] == 'a1'
-    assert d[0][1].type is BLong
+    for extra_args in [(SF_PACKED,), (0, 1)]:
+        BStruct = new_struct_type("struct foo")
+        complete_struct_or_union(BStruct, [('a1', BLong, -1),
+                                           ('a2', BChar, -1),
+                                           ('a3', BShort, -1)],
+                                 None, -1, -1, *extra_args)
+        d = BStruct.fields
+        assert len(d) == 3
+        assert d[0][0] == 'a1'
+        assert d[0][1].type is BLong
+        assert d[0][1].offset == 0
+        assert d[0][1].bitshift == -1
+        assert d[0][1].bitsize == -1
+        assert d[1][0] == 'a2'
+        assert d[1][1].type is BChar
+        assert d[1][1].offset == sizeof(BLong)
+        assert d[1][1].bitshift == -1
+        assert d[1][1].bitsize == -1
+        assert d[2][0] == 'a3'
+        assert d[2][1].type is BShort
+        assert d[2][1].offset == sizeof(BLong) + sizeof(BChar)
+        assert d[2][1].bitshift == -1
+        assert d[2][1].bitsize == -1
+        assert sizeof(BStruct) == sizeof(BLong) + sizeof(BChar) + sizeof(BShort)
+        assert alignof(BStruct) == 1
+    #
+    BStruct2 = new_struct_type("struct foo")
+    complete_struct_or_union(BStruct2, [('b1', BChar, -1),
+                                        ('b2', BLong, -1)],
+                             None, -1, -1, 0, 2)
+    d = BStruct2.fields
+    assert len(d) == 2
+    assert d[0][0] == 'b1'
+    assert d[0][1].type is BChar
     assert d[0][1].offset == 0
     assert d[0][1].bitshift == -1
     assert d[0][1].bitsize == -1
-    assert d[1][0] == 'a2'
-    assert d[1][1].type is BChar
-    assert d[1][1].offset == sizeof(BLong)
+    assert d[1][0] == 'b2'
+    assert d[1][1].type is BLong
+    assert d[1][1].offset == 2
     assert d[1][1].bitshift == -1
     assert d[1][1].bitsize == -1
-    assert d[2][0] == 'a3'
-    assert d[2][1].type is BShort
-    assert d[2][1].offset == sizeof(BLong) + sizeof(BChar)
-    assert d[2][1].bitshift == -1
-    assert d[2][1].bitsize == -1
-    assert sizeof(BStruct) == sizeof(BLong) + sizeof(BChar) + sizeof(BShort)
-    assert alignof(BStruct) == 1
+    assert sizeof(BStruct2) == 2 + sizeof(BLong)
+    assert alignof(BStruct2) == 2
 
 def test_packed_with_bitfields():
     if sys.platform == "win32":
@@ -3915,7 +3935,7 @@
 
 def test_char_pointer_conversion():
     import warnings
-    assert __version__.startswith(("1.8", "1.9", "1.10", "1.11")), (
+    assert __version__.startswith(("1.8", "1.9", "1.10", "1.11", "1.12")), (
         "consider turning the warning into an error")
     BCharP = new_pointer_type(new_primitive_type("char"))
     BIntP = new_pointer_type(new_primitive_type("int"))
diff --git a/pypy/module/_cppyy/interp_cppyy.py b/pypy/module/_cppyy/interp_cppyy.py
--- a/pypy/module/_cppyy/interp_cppyy.py
+++ b/pypy/module/_cppyy/interp_cppyy.py
@@ -455,7 +455,7 @@
 class MethodWithProps(Method):
     # allow user to determine ffi use rules per overload
     def fget_useffi(self, space):
-	f = space.interp_w(W_CPPOverload, self.w_function)
+        f = space.interp_w(W_CPPOverload, self.w_function)
         return f.fget_useffi(space)
 
     @unwrap_spec(value=bool)
@@ -623,7 +623,7 @@
             # onto a class and w_this should be set
             cppinstance = self.space.interp_w(W_CPPInstance, w_obj)
             if cppinstance.clsdecl.handle != self.scope.handle:
-	        return MethodWithProps(self.space, self, w_obj)    # bound
+                return MethodWithProps(self.space, self, w_obj)    # bound
         return self      # unbound
 
     @unwrap_spec(args_w='args_w')
@@ -764,7 +764,7 @@
                 self.master.overloads[fullname] = method
 
         if not self.space.is_w(self.w_this, self.space.w_None):
-	    return self.space.call_obj_args(method, self.w_this, Arguments(self.space, args_w))
+            return self.space.call_obj_args(method, self.w_this, Arguments(self.space, args_w))
         return self.space.call_args(method, Arguments(self.space, args_w))
 
     def getitem_impl(self, name, args_w):
@@ -803,10 +803,10 @@
          self.name = name
          self.overloads = {}
          self.master = self
-	 self.w_this = space.w_None
+         self.w_this = space.w_None
 
     def descr_get(self, w_cppinstance, w_cls=None):
-	# TODO: don't return copy, but bind in an external object (like W_CPPOverload)
+        # TODO: don't return copy, but bind in an external object (like W_CPPOverload)
         if self.space.is_w(w_cppinstance, self.space.w_None):
             return self  # unbound, so no new instance needed
         cppol = W_CPPTemplateOverload(self.space, self.name, self.scope, self.functions, self.flags)
@@ -858,10 +858,10 @@
          self.name = name
          self.overloads = {}
          self.master = self
-	 self.w_this = space.w_None
+         self.w_this = space.w_None
 
     def descr_get(self, w_cppinstance, w_cls=None):
-	# TODO: don't return copy, but bind in an external object (like W_CPPOverload)
+        # TODO: don't return copy, but bind in an external object (like W_CPPOverload)
         if isinstance(w_cppinstance, W_CPPInstance):
             cppinstance = self.space.interp_w(W_CPPInstance, w_cppinstance)
             if cppinstance.clsdecl.handle != self.scope.handle:
diff --git a/pypy/module/cpyext/cdatetime.py b/pypy/module/cpyext/cdatetime.py
--- a/pypy/module/cpyext/cdatetime.py
+++ b/pypy/module/cpyext/cdatetime.py
@@ -120,21 +120,10 @@
                    dealloc=type_dealloc,
                   )
 
-    # why do we need date_dealloc? Since W_DateTime_Date is the base class for
-    # app level datetime.date. If a c-extension class uses datetime.date for its
-    # base class and defines a tp_dealloc, we will get this:
-    # c_class->tp_dealloc == tp_dealloc_func
-    # c_class->tp_base == datetime.date,
-    #                     datetime.date->tp_dealloc = _PyPy_subtype_dealloc
-    # datetime.date->tp_base = W_DateTime_Date
-    #                    W_DateTime_Date->tp_dealloc = _PyPy_subtype_dealloc
-    # but _PyPy_subtype_dealloc will call tp_dealloc_func, which can call its
-    # base's tp_dealloc and we get recursion. So break the recursion by setting
-    # W_DateTime_Date->tp_dealloc
     make_typedescr(W_DateTime_Date.typedef,
                    basestruct=PyDateTime_DateTime.TO,
                    attach=type_attach,
-                   dealloc=date_dealloc,
+                   dealloc=type_dealloc,
                   )
 
     make_typedescr(W_DateTime_Delta.typedef,
@@ -144,30 +133,41 @@
 
 def type_attach(space, py_obj, w_obj, w_userdata=None):
     '''Fills a newly allocated py_obj from the w_obj
+    If it is a datetime.time or datetime.datetime, it may have tzinfo
     '''
-    if space.type(w_obj).name == 'date':
-        # No tzinfo
-        return
-    py_datetime = rffi.cast(PyDateTime_Time, py_obj)
-    w_tzinfo = space.getattr(w_obj, space.newtext('_tzinfo'))
-    if space.is_none(w_tzinfo):
-        py_datetime.c_hastzinfo = cts.cast('unsigned char', 0)
-        py_datetime.c_tzinfo = lltype.nullptr(PyObject.TO)
-    else:
-        py_datetime.c_hastzinfo = cts.cast('unsigned char', 1)
-        py_datetime.c_tzinfo = make_ref(space, w_tzinfo)
+    assert len(datetimeAPI_global) > 0
+    if datetimeAPI_global[0].c_TimeType == py_obj.c_ob_type:
+        py_datetime = rffi.cast(PyDateTime_Time, py_obj)
+        w_tzinfo = space.getattr(w_obj, space.newtext('tzinfo'))
+        if space.is_none(w_tzinfo):
+            py_datetime.c_hastzinfo = cts.cast('unsigned char', 0)
+            py_datetime.c_tzinfo = lltype.nullptr(PyObject.TO)
+        else:
+            py_datetime.c_hastzinfo = cts.cast('unsigned char', 1)
+            py_datetime.c_tzinfo = make_ref(space, w_tzinfo)
+    elif datetimeAPI_global[0].c_DateTimeType == py_obj.c_ob_type:
+        # For now this is exactly the same structure as PyDateTime_Time
+        py_datetime = rffi.cast(PyDateTime_DateTime, py_obj)
+        w_tzinfo = space.getattr(w_obj, space.newtext('tzinfo'))
+        if space.is_none(w_tzinfo):
+            py_datetime.c_hastzinfo = cts.cast('unsigned char', 0)
+            py_datetime.c_tzinfo = lltype.nullptr(PyObject.TO)
+        else:
+            py_datetime.c_hastzinfo = cts.cast('unsigned char', 1)
+            py_datetime.c_tzinfo = make_ref(space, w_tzinfo)
 
 @slot_function([PyObject], lltype.Void)
 def type_dealloc(space, py_obj):
-    py_datetime = rffi.cast(PyDateTime_Time, py_obj)
-    if (widen(py_datetime.c_hastzinfo) != 0):
-        decref(space, py_datetime.c_tzinfo)
     from pypy.module.cpyext.object import _dealloc
-    _dealloc(space, py_obj)
-
- at slot_function([PyObject], lltype.Void)
-def date_dealloc(space, py_obj):
-    from pypy.module.cpyext.object import _dealloc
+    assert len(datetimeAPI_global) > 0
+    if datetimeAPI_global[0].c_TimeType == py_obj.c_ob_type:
+        py_datetime = rffi.cast(PyDateTime_Time, py_obj)
+        if (widen(py_datetime.c_hastzinfo) != 0):
+            decref(space, py_datetime.c_tzinfo)
+    elif datetimeAPI_global[0].c_DateTimeType == py_obj.c_ob_type:
+        py_datetime = rffi.cast(PyDateTime_DateTime, py_obj)
+        if (widen(py_datetime.c_hastzinfo) != 0):
+            decref(space, py_datetime.c_tzinfo)
     _dealloc(space, py_obj)
 
 def timedeltatype_attach(space, py_obj, w_obj, w_userdata=None):
diff --git a/pypy/module/cpyext/test/test_datetime.py b/pypy/module/cpyext/test/test_datetime.py
--- a/pypy/module/cpyext/test/test_datetime.py
+++ b/pypy/module/cpyext/test/test_datetime.py
@@ -307,18 +307,20 @@
         from datetime import tzinfo, datetime, timedelta, time
         # copied from datetime documentation
         class GMT1(tzinfo):
-           def utcoffset(self, dt):
-               return timedelta(hours=1) + self.dst(dt)
-           def dst(self, dt):
-               return timedelta(0)
-           def tzname(self,dt):
+            def __del__(self):
+                print 'deleting GMT1'
+            def utcoffset(self, dt):
+                return timedelta(hours=1) + self.dst(dt)
+            def dst(self, dt):
+                return timedelta(0)
+            def tzname(self,dt):
                 return "GMT +1"
         gmt1 = GMT1()
         dt1 = module.time_with_tzinfo(gmt1)
         assert dt1 == time(6, 6, 6, 6, gmt1)
         assert '+01' in str(dt1)
-        assert module.datetime_with_tzinfo(gmt1) == datetime(
-            2000, 6, 6, 6, 6, 6, 6, gmt1)
+        dt_tz = module.datetime_with_tzinfo(gmt1)
+        assert dt_tz == datetime(2000, 6, 6, 6, 6, 6, 6, gmt1)
 
     def test_checks(self):
         module = self.import_extension('foo', [
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py b/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py
@@ -1824,19 +1824,39 @@
         ffi = FFI(backend=self.Backend())
         ffi.cdef("struct nonpacked { char a; int b; };")
         ffi.cdef("struct is_packed { char a; int b; };", packed=True)
+        ffi.cdef("struct is_packed1 { char a; int b; };", pack=1)
+        ffi.cdef("struct is_packed2 { char a; int b; };", pack=2)
+        ffi.cdef("struct is_packed4 { char a; int b; };", pack=4)
+        ffi.cdef("struct is_packed8 { char a; int b; };", pack=8)
         assert ffi.sizeof("struct nonpacked") == 8
         assert ffi.sizeof("struct is_packed") == 5
+        assert ffi.sizeof("struct is_packed1") == 5
+        assert ffi.sizeof("struct is_packed2") == 6
+        assert ffi.sizeof("struct is_packed4") == 8
+        assert ffi.sizeof("struct is_packed8") == 8
         assert ffi.alignof("struct nonpacked") == 4
         assert ffi.alignof("struct is_packed") == 1
-        s = ffi.new("struct is_packed[2]")
-        s[0].b = 42623381
-        s[0].a = b'X'
-        s[1].b = -4892220
-        s[1].a = b'Y'
-        assert s[0].b == 42623381
-        assert s[0].a == b'X'
-        assert s[1].b == -4892220
-        assert s[1].a == b'Y'
+        assert ffi.alignof("struct is_packed1") == 1
+        assert ffi.alignof("struct is_packed2") == 2
+        assert ffi.alignof("struct is_packed4") == 4
+        assert ffi.alignof("struct is_packed8") == 4
+        for name in ['is_packed', 'is_packed1', 'is_packed2',
+                     'is_packed4', 'is_packed8']:
+            s = ffi.new("struct %s[2]" % name)
+            s[0].b = 42623381
+            s[0].a = b'X'
+            s[1].b = -4892220
+            s[1].a = b'Y'
+            assert s[0].b == 42623381
+            assert s[0].a == b'X'
+            assert s[1].b == -4892220
+            assert s[1].a == b'Y'
+
+    def test_pack_valueerror(self):
+        ffi = FFI(backend=self.Backend())
+        py.test.raises(ValueError, ffi.cdef, "", pack=3)
+        py.test.raises(ValueError, ffi.cdef, "", packed=2)
+        py.test.raises(ValueError, ffi.cdef, "", packed=True, pack=1)
 
     def test_define_integer_constant(self):
         ffi = FFI(backend=self.Backend())
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
@@ -2243,6 +2243,12 @@
         "'API mode' and non-variadic (i.e. declared inside ffibuilder.cdef()"
         "+ffibuilder.set_source() and not taking a final '...' argument)")
 
+def test_pack_not_supported():
+    ffi = FFI()
+    ffi.cdef("""struct foo { char y; int x; };""", pack=2)
+    py.test.raises(NotImplementedError, verify,
+                   ffi, "test_pack_not_supported", "")
+
 def test_gcc_visibility_hidden():
     if sys.platform == 'win32':
         py.test.skip("test for gcc/clang")

From pypy.commits at gmail.com  Fri Jul 27 16:02:20 2018
From: pypy.commits at gmail.com (mattip)
Date: Fri, 27 Jul 2018 13:02:20 -0700 (PDT)
Subject: [pypy-commit] pypy unicode-utf8-py3: fix, skip tests
Message-ID: <5b5b7a4c.1c69fb81.2ac30.30db@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: unicode-utf8-py3
Changeset: r94902:74c2ede43307
Date: 2018-07-11 23:32 -0500
http://bitbucket.org/pypy/pypy/changeset/74c2ede43307/

Log:	fix, skip tests

diff --git a/pypy/interpreter/astcompiler/test/test_astbuilder.py b/pypy/interpreter/astcompiler/test/test_astbuilder.py
--- a/pypy/interpreter/astcompiler/test/test_astbuilder.py
+++ b/pypy/interpreter/astcompiler/test/test_astbuilder.py
@@ -902,7 +902,7 @@
 
     def test_flufl(self):
         source = "x <> y"
-        raises(SyntaxError, self.get_ast, source)
+        py.test.raises(SyntaxError, self.get_ast, source)
         comp = self.get_first_expr(source,
                                    flags=consts.CO_FUTURE_BARRY_AS_BDFL)
         assert isinstance(comp, ast.Compare)
@@ -1130,7 +1130,7 @@
         s = self.get_first_expr("b'hi' b' implicitly' b' extra'")
         assert isinstance(s, ast.Bytes)
         assert space.eq_w(s.s, space.newbytes("hi implicitly extra"))
-        raises(SyntaxError, self.get_first_expr, "b'hello' 'world'")
+        py.test.raises(SyntaxError, self.get_first_expr, "b'hello' 'world'")
         sentence = u"Die M&#228;nner &#228;rgern sich!"
         source = u"# coding: utf-7\nstuff = '%s'" % (sentence,)
         info = pyparse.CompileInfo("<test>", "exec")
@@ -1325,8 +1325,8 @@
         assert isinstance(if2, ast.Name)
 
     def test_cpython_issue12983(self):
-        raises(SyntaxError, self.get_ast, r"""b'\x'""")
-        raises(SyntaxError, self.get_ast, r"""b'\x0'""")
+        py.test.raises(SyntaxError, self.get_ast, r"""b'\x'""")
+        py.test.raises(SyntaxError, self.get_ast, r"""b'\x0'""")
 
     def test_matmul(self):
         mod = self.get_ast("a @ b")
diff --git a/pypy/interpreter/test/test_unicodehelper.py b/pypy/interpreter/test/test_unicodehelper.py
--- a/pypy/interpreter/test/test_unicodehelper.py
+++ b/pypy/interpreter/test/test_unicodehelper.py
@@ -78,6 +78,7 @@
     assert lst == [("??", "ascii", input, 0, 2),
                    ("??", "ascii", input, 5, 7)]
 
+ at pytest.skip("rework this test for utf8")
 def test_decode_utf8_allow_surrogates():
     sp = FakeSpace()
     assert decode_utf8(sp, "\xed\xa0\x80", allow_surrogates=True) == u"\ud800"
@@ -87,6 +88,7 @@
     got = decode_utf8(sp, "\xf0\x90\x80\x80", allow_surrogates=True)
     assert map(ord, got) == [0x10000]
 
+ at pytest.skip("rework this test for utf8")
 def test_decode_utf8sp():
     space = FakeSpace()
     assert decode_utf8sp(space, "\xed\xa0\x80") == u"\ud800"
@@ -96,6 +98,7 @@
     got = decode_utf8sp(space, "\xf0\x90\x80\x80")
     assert map(ord, got) == [0x10000]
 
+ at pytest.skip("test has non-valid errorhandler")
 @pytest.mark.parametrize('unich', [u"\ud800", u"\udc80"])
 def test_utf32_surrogates(unich):
     assert (utf8_encode_utf_32_be(unich.encode('utf-8'), None) ==
@@ -108,10 +111,12 @@
                               allow_surrogates=False)
 
     def replace_with(ru, rs):
+        if rs:
+            rs = rs.encode('utf-8')
         def errorhandler(errors, enc, msg, u, startingpos, endingpos):
             if errors == 'strict':
                 raise UnicodeEncodeError(enc, u, startingpos, endingpos, msg)
-            return ru.encode('utf-8'), endingpos
+            return rs, endingpos
         uch = u"<%s>" % unich
         return utf8_encode_utf_32_be(
             uch.encode('utf8'), None,
diff --git a/pypy/module/_codecs/test/test_codecs.py b/pypy/module/_codecs/test/test_codecs.py
--- a/pypy/module/_codecs/test/test_codecs.py
+++ b/pypy/module/_codecs/test/test_codecs.py
@@ -15,7 +15,7 @@
                          'utf-32', 'utf-32-le', 'utf-32-be',
                          'raw_unicode_escape',
                          'unicode_escape', 'unicode_internal'):
-            assert str(u.encode(encoding),encoding) == u
+            assert str(u.encode(encoding), encoding) == u
 
     def test_ucs4(self):
         x = u'\U00100000'

From pypy.commits at gmail.com  Fri Jul 27 16:02:22 2018
From: pypy.commits at gmail.com (mattip)
Date: Fri, 27 Jul 2018 13:02:22 -0700 (PDT)
Subject: [pypy-commit] pypy unicode-utf8-py3: remove consumed from decoding
 functions
Message-ID: <5b5b7a4e.1c69fb81.80661.b8e2@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: unicode-utf8-py3
Changeset: r94903:97cd2e230f8a
Date: 2018-07-11 23:33 -0500
http://bitbucket.org/pypy/pypy/changeset/97cd2e230f8a/

Log:	remove consumed from decoding functions

diff --git a/pypy/interpreter/unicodehelper.py b/pypy/interpreter/unicodehelper.py
--- a/pypy/interpreter/unicodehelper.py
+++ b/pypy/interpreter/unicodehelper.py
@@ -31,7 +31,7 @@
 
 def decode_never_raise(errors, encoding, msg, s, startingpos, endingpos):
     ux = ['\ux' + hex(ord(x))[2:].upper() for x in s[startingpos:endingpos]]
-    return ''.join(ux), endingpos
+    return ''.join(ux), endingpos, endingpos
 
 @specialize.memo()
 def encode_error_handler(space):
@@ -144,7 +144,7 @@
     from pypy.module._codecs import interp_codecs
     state = space.fromcache(interp_codecs.CodecState)
     unicodedata_handler = state.get_unicodedata_handler(space)
-    result_utf8, consumed, length = str_decode_unicode_escape(
+    result_utf8, length = str_decode_unicode_escape(
         string, "strict",
         final=True,
         errorhandler=decode_error_handler(space),
@@ -152,7 +152,7 @@
     return result_utf8, length
 
 def decode_raw_unicode_escape(space, string):
-    result_utf8, consumed, lgt = str_decode_raw_unicode_escape(
+    result_utf8, lgt = str_decode_raw_unicode_escape(
         string, "strict",
         final=True, errorhandler=decode_error_handler(space))
     return result_utf8, lgt
@@ -172,7 +172,7 @@
     # If there happen to be two 3-bytes encoding a pair of surrogates,
     # you still get two surrogate unicode characters in the result.
     assert isinstance(string, str)
-    result, consumed = runicode.str_decode_utf_8(
+    result, lgth = runicode.str_decode_utf_8(
         string, len(string), "strict",
         final=True, errorhandler=decode_error_handler(space),
         # XXX handle surrogates
@@ -182,7 +182,7 @@
 def str_decode_ascii(s, errors, final, errorhandler):
     try:
         rutf8.check_ascii(s)
-        return s, len(s), len(s)
+        return s, len(s)
     except rutf8.CheckError:
         return _str_decode_ascii_slowpath(s, errors, final, errorhandler)
 
@@ -200,12 +200,12 @@
             i += 1
     ress = res.build()
     lgt = rutf8.check_utf8(ress, True)
-    return ress, len(s), lgt
+    return ress, lgt
 
 def str_decode_latin_1(s, errors, final, errorhandler):
     try:
         rutf8.check_ascii(s)
-        return s, len(s), len(s)
+        return s, len(s)
     except rutf8.CheckError:
         return _str_decode_latin_1_slowpath(s, errors, final, errorhandler)
 
@@ -225,7 +225,7 @@
             res.append_slice(s, start, end)
             i = end
     # cannot be ASCII, cannot have surrogates, I believe
-    return res.build(), len(s), len(s)
+    return res.build(), len(s)
 
 def utf8_encode_latin_1(s, errors, errorhandler):
     try:
@@ -310,7 +310,7 @@
         slen = len(s)
         res, size = runicode.str_decode_mbcs(s, slen, final=final, errors=errors,
                                            errorhandler=errorhandler)
-        return res.encode('utf8'), size, len(res)
+        return res.encode('utf8'), len(res)
 
 def str_decode_utf8(s, errors, final, errorhandler, allow_surrogates=False):
     """ Same as checking for the valid utf8, but we know the utf8 is not
@@ -432,7 +432,7 @@
         res.append(r)
 
     r = res.build()
-    return r, pos, rutf8.check_utf8(r, True)
+    return r, rutf8.check_utf8(r, True)
 
 hexdigits = "0123456789ABCDEFabcdef"
 
@@ -471,7 +471,7 @@
 def str_decode_unicode_escape(s, errors, final, errorhandler, ud_handler):
     size = len(s)
     if size == 0:
-        return '', 0, 0
+        return '', 0
 
     builder = rutf8.Utf8StringBuilder(size)
     pos = 0
@@ -590,7 +590,7 @@
             builder.append_char('\\')
             builder.append_code(ord(ch))
 
-    return builder.build(), pos, builder.getlength()
+    return builder.build(), builder.getlength()
 
 def wcharpsize2utf8(space, wcharp, size):
     """Safe version of rffi.wcharpsize2utf8.
@@ -612,7 +612,7 @@
                                   errorhandler=None):
     size = len(s)
     if size == 0:
-        return '', 0, 0
+        return '', 0
 
     builder = rutf8.Utf8StringBuilder(size)
     pos = 0
@@ -652,7 +652,7 @@
         pos = hexescape(builder, s, pos, digits,
                            "rawunicodeescape", errorhandler, message, errors)
 
-    return builder.build(), pos, builder.getlength()
+    return builder.build(), builder.getlength()
 
 _utf8_encode_unicode_escape = rutf8.make_utf8_escape_function()
 
@@ -787,7 +787,7 @@
                      errorhandler=None):
     size = len(s)
     if size == 0:
-        return '', 0, 0
+        return '', 0
 
     inShift = False
     base64bits = 0
@@ -922,7 +922,7 @@
         final_length = shiftOutStartPos # back off output
 
     assert final_length >= 0
-    return result.build()[:final_length], pos, outsize
+    return result.build()[:final_length], outsize
 
 def utf8_encode_utf_7(s, errors, errorhandler):
     size = len(s)
@@ -1012,21 +1012,21 @@
 
 def str_decode_utf_16(s, errors, final=True,
                       errorhandler=None):
-    result, c, lgt, _ = str_decode_utf_16_helper(s, errors, final,
+    result, lgt = str_decode_utf_16_helper(s, errors, final,
                                                          errorhandler, "native")
-    return result, c, lgt
+    return result, lgt
 
 def str_decode_utf_16_be(s, errors, final=True,
                         errorhandler=None):
-    result, c, lgt, _ = str_decode_utf_16_helper(s, errors, final,
+    result, lgt = str_decode_utf_16_helper(s, errors, final,
                                                          errorhandler, "big")
-    return result, c, lgt
+    return result, lgt
 
 def str_decode_utf_16_le(s, errors, final=True,
                          errorhandler=None):
-    result, c, lgt, _ = str_decode_utf_16_helper(s, errors, final,
+    result, lgt = str_decode_utf_16_helper(s, errors, final,
                                                          errorhandler, "little")
-    return result, c, lgt
+    return result, lgt
 
 def str_decode_utf_16_helper(s, errors, final=True,
                              errorhandler=None,
@@ -1069,7 +1069,7 @@
     else:
         bo = 1
     if size == 0:
-        return '', 0, 0, bo
+        return '', 0
     if bo == -1:
         # force little endian
         ihi = 1
@@ -1129,7 +1129,7 @@
             result.append(r)
     r = result.build()
     lgt = rutf8.check_utf8(r, True)
-    return result.build(), pos, lgt, bo
+    return result.build(), lgt
 
 def _STORECHAR(result, CH, byteorder):
     hi = chr(((CH) >> 8) & 0xff)
@@ -1162,8 +1162,26 @@
     pos = 0
     index = 0
     while pos < size:
-        ch = rutf8.codepoint_at_pos(s, pos)
-
+        try:
+            ch = rutf8.codepoint_at_pos(s, pos)
+        except IndexError:
+            # malformed codepoint, blindly use ch
+            ch = ord(s[pos])
+            pos += 1
+            if errorhandler:
+                res_8, newindex = errorhandler(
+                    errors, public_encoding_name, 'malformed unicode',
+                    s, pos - 1, pos)
+                for cp in rutf8.Utf8StringIterator(res_8):
+                    if cp < 0xD800:
+                        _STORECHAR(result, cp, byteorder)
+                    else:
+                        errorhandler('strict', public_encoding_name,
+                                     'malformed unicode',
+                                 s, pos-1, pos)
+            else:
+                _STORECHAR(result, ch, byteorder)
+                continue
         if ch < 0xD800:
             _STORECHAR(result, ch, byteorder)
         elif ch >= 0x10000:
@@ -1219,21 +1237,21 @@
 
 def str_decode_utf_32(s, errors, final=True,
                       errorhandler=None):
-    result, c, lgt, _ = str_decode_utf_32_helper(
+    result, lgt_ = str_decode_utf_32_helper(
         s, errors, final, errorhandler, "native", 'utf-32-' + BYTEORDER2,
         allow_surrogates=False)
-    return result, c, lgt
+    return result, lgt
 
 def str_decode_utf_32_be(s, errors, final=True,
                          errorhandler=None):
-    result, c, lgt, _ = str_decode_utf_32_helper(
+    result, lgt = str_decode_utf_32_helper(
         s, errors, final, errorhandler, "big", 'utf-32-be',
         allow_surrogates=False)
-    return result, c, lgt
+    return result, lgt
 
 def str_decode_utf_32_le(s, errors, final=True,
                          errorhandler=None):
-    result, c, lgt, _ = str_decode_utf_32_helper(
+    result, lgt_ = str_decode_utf_32_helper(
         s, errors, final, errorhandler, "little", 'utf-32-le',
         allow_surrogates=False)
     return result, c, lgt
@@ -1284,7 +1302,7 @@
     else:
         bo = 1
     if size == 0:
-        return '', 0, 0, bo
+        return '', 0
     if bo == -1:
         # force little endian
         iorder = [0, 1, 2, 3]
@@ -1326,7 +1344,7 @@
         pos += 4
     r = result.build()
     lgt = rutf8.check_utf8(r, True)
-    return r, pos, lgt, bo
+    return r, lgt
 
 def _STORECHAR32(result, CH, byteorder):
     c0 = chr(((CH) >> 24) & 0xff)
@@ -1365,8 +1383,31 @@
     pos = 0
     index = 0
     while pos < size:
-        ch = rutf8.codepoint_at_pos(s, pos)
-        pos = rutf8.next_codepoint_pos(s, pos)
+        try:
+            ch = rutf8.codepoint_at_pos(s, pos)
+            pos = rutf8.next_codepoint_pos(s, pos)
+        except IndexError:
+            # malformed codepoint, blindly use ch
+            ch = ord(s[pos])
+            pos += 1
+            if errorhandler:
+                res_8, newindex = errorhandler(
+                    errors, public_encoding_name, 'malformed unicode',
+                    s, pos - 1, pos)
+                if res_8:
+                    for cp in rutf8.Utf8StringIterator(res_8):
+                        if cp < 0xD800:
+                            _STORECHAR32(result, cp, byteorder)
+                        else:
+                            errorhandler('strict', public_encoding_name,
+                                     'malformed unicode',
+                                 s, pos-1, pos)
+                else:
+                    _STORECHAR32(result, ch, byteorder)
+            else:
+                _STORECHAR32(result, ch, byteorder)
+            index += 1
+            continue
         if not allow_surrogates and 0xD800 <= ch < 0xE000:
             res_8, newindex = errorhandler(
                 errors, public_encoding_name, 'surrogates not allowed',
@@ -1389,19 +1430,19 @@
 
 def utf8_encode_utf_32(s, errors,
                           errorhandler=None, allow_surrogates=True):
-    return unicode_encode_utf_32_helper(s, errors, errorhandler,
+    return unicode_encode_utf_32_helper(s.decode('utf8'), errors, errorhandler,
                                         allow_surrogates, "native",
                                         'utf-32-' + BYTEORDER2)
 
 def utf8_encode_utf_32_be(s, errors,
                                   errorhandler=None, allow_surrogates=True):
-    return unicode_encode_utf_32_helper(s, errors, errorhandler,
+    return unicode_encode_utf_32_helper(s.decode('utf8'), errors, errorhandler,
                                         allow_surrogates, "big",
                                         'utf-32-be')
 
 def utf8_encode_utf_32_le(s, errors,
                                   errorhandler=None, allow_surrogates=True):
-    return unicode_encode_utf_32_helper(s, errors, errorhandler,
+    return unicode_encode_utf_32_helper(s.decode('utf8'), errors, errorhandler,
                                         allow_surrogates, "little",
                                         'utf-32-le')
 # ____________________________________________________________
@@ -1411,7 +1452,7 @@
                                 errorhandler=None):
     size = len(s)
     if size == 0:
-        return '', 0, 0
+        return '', 0
 
     unicode_bytes = 4
     if BYTEORDER == "little":
@@ -1449,7 +1490,7 @@
         pos += unicode_bytes
     r = result.build()
     lgt = rutf8.check_utf8(r, True)
-    return r, pos, lgt
+    return r, lgt
 
 def utf8_encode_unicode_internal(s, errors, errorhandler):
     size = len(s)
@@ -1490,7 +1531,7 @@
                                   errorhandler=errorhandler)
     size = len(s)
     if size == 0:
-        return '', 0, 0
+        return '', 0
 
     pos = 0
     result = StringBuilder(size)
@@ -1508,7 +1549,7 @@
         pos += 1
     r = result.build()
     lgt = rutf8.check_utf8(r, True)
-    return r, pos, lgt
+    return r, lgt
 
 def utf8_encode_charmap(s, errors, errorhandler=None, mapping=None):
     size = len(s)
diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py
--- a/pypy/objspace/std/objspace.py
+++ b/pypy/objspace/std/objspace.py
@@ -384,10 +384,10 @@
         if isinstance(s, unicode):
             s, lgt = s.encode('utf8'), len(s)
         elif isinstance(s, str):
-            s, uf8lgt, lgt = decode_utf8sp(self, s)
+            s, lgt = decode_utf8sp(self, s)
         elif isinstance(s, tuple):
             # result of decode_utf8
-            s, utf8lgt, lgt = s
+            s, lgt = s
         else:
             # XXX what is s ?
             lgt = rutf8.check_utf8(s, True)

From pypy.commits at gmail.com  Fri Jul 27 16:02:25 2018
From: pypy.commits at gmail.com (mattip)
Date: Fri, 27 Jul 2018 13:02:25 -0700 (PDT)
Subject: [pypy-commit] pypy unicode-utf8-py3: fix tests,
 then fix implementations to pass more tests
Message-ID: <5b5b7a51.1c69fb81.bdb68.d818@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: unicode-utf8-py3
Changeset: r94904:fed03b2241ff
Date: 2018-07-27 16:00 -0400
http://bitbucket.org/pypy/pypy/changeset/fed03b2241ff/

Log:	fix tests, then fix implementations to pass more tests

diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -1721,7 +1721,9 @@
         return w_obj.convert_to_w_unicode(self)
 
     def realunicode_w(self, w_obj):
-        return w_obj.realunicode_w(self)
+        from rpython.rlib.runicode import str_decode_utf_8
+        utf8 = self.utf8_w(w_obj)
+        return str_decode_utf_8(utf8, len(utf8), 'strict', True)[0]
 
     def utf8_0_w(self, w_obj):
         "Like utf8_w, but rejects strings with NUL bytes."
diff --git a/pypy/interpreter/error.py b/pypy/interpreter/error.py
--- a/pypy/interpreter/error.py
+++ b/pypy/interpreter/error.py
@@ -472,16 +472,16 @@
     assert len(formats) > 0, "unsupported: no % command found"
     return tuple(parts), tuple(formats)
 
- at specialize.arg(1)
 def _decode_utf8(string):
     # when building the error message, don't crash if the byte string
     # provided is not valid UTF-8
     if isinstance(string, unicode):
         return string
     assert isinstance(string, str)
-    result, consumed = runicode.str_decode_utf_8(
-        string, len(string), "replace", final=True)
-    return result
+    return string.decode('utf8')
+    #result, consumed = runicode.str_decode_utf_8(
+    #    string, len(string), "replace", final=True)
+    #return result
 
 def get_operrcls2(valuefmt):
     valuefmt = valuefmt.decode('ascii')
diff --git a/pypy/interpreter/unicodehelper.py b/pypy/interpreter/unicodehelper.py
--- a/pypy/interpreter/unicodehelper.py
+++ b/pypy/interpreter/unicodehelper.py
@@ -147,7 +147,7 @@
     result_utf8, length = str_decode_unicode_escape(
         string, "strict",
         final=True,
-        errorhandler=decode_error_handler(space),
+        errorhandler=state.decode_error_handler,
         ud_handler=unicodedata_handler)
     return result_utf8, length
 
@@ -161,7 +161,6 @@
     try:
         rutf8.check_ascii(string)
     except rutf8.CheckError as e:
-        print 'check_ascii_or_raise', string
         decode_error_handler(space)('strict', 'ascii',
                                     'ordinal not in range(128)', string,
                                     e.pos, e.pos + 1)
@@ -172,12 +171,12 @@
     # If there happen to be two 3-bytes encoding a pair of surrogates,
     # you still get two surrogate unicode characters in the result.
     assert isinstance(string, str)
-    result, lgth = runicode.str_decode_utf_8(
-        string, len(string), "strict",
-        final=True, errorhandler=decode_error_handler(space),
-        # XXX handle surrogates
-        allow_surrogates=False)
-    return len(result)
+    try:
+        return rutf8.check_utf8(string, True, start, end)
+    except rutf8.CheckError as e:
+        decode_error_handler(space)('strict', 'utf8',
+                                    'unexpected end of data', string,
+                                    e.pos, e.pos + 1)
 
 def str_decode_ascii(s, errors, final, errorhandler):
     try:
@@ -298,10 +297,9 @@
     return result.build()
 
 if sys.platform == 'win32':
-    def utf8_encode_mbcs(s, errors, errorhandler):
+    def utf8_encode_mbcs(s, slen, errors, errorhandler):
         from rpython.rlib import runicode
         s = s.decode('utf-8')
-        slen = len(s)
         res = runicode.unicode_encode_mbcs(s, slen, errors, errorhandler)
         return res
         
@@ -647,7 +645,7 @@
             continue
 
         digits = 4 if s[pos] == 'u' else 8
-        message = "truncated \\uXXXX"
+        message = "truncated \\uXXXX escape"
         pos += 1
         pos = hexescape(builder, s, pos, digits,
                            "rawunicodeescape", errorhandler, message, errors)
@@ -1141,7 +1139,7 @@
         result.append(hi)
         result.append(lo)
 
-def unicode_encode_utf_16_helper(s, errors,
+def utf8_encode_utf_16_helper(s, errors,
                                  errorhandler=None,
                                  allow_surrogates=True,
                                  byteorder='little',
@@ -1213,21 +1211,21 @@
 def utf8_encode_utf_16(s, errors,
                           errorhandler=None,
                           allow_surrogates=True):
-    return unicode_encode_utf_16_helper(s, errors, errorhandler,
+    return utf8_encode_utf_16_helper(s, errors, errorhandler,
                                         allow_surrogates, "native",
                                         'utf-16-' + BYTEORDER2)
 
 def utf8_encode_utf_16_be(s, errors,
                              errorhandler=None,
                              allow_surrogates=True):
-    return unicode_encode_utf_16_helper(s, errors, errorhandler,
+    return utf8_encode_utf_16_helper(s, errors, errorhandler,
                                         allow_surrogates, "big",
                                         'utf-16-be')
 
 def utf8_encode_utf_16_le(s, errors,
                              errorhandler=None,
                              allow_surrogates=True):
-    return unicode_encode_utf_16_helper(s, errors, errorhandler,
+    return utf8_encode_utf_16_helper(s, errors, errorhandler,
                                         allow_surrogates, "little",
                                         'utf-16-le')
 
@@ -1237,7 +1235,7 @@
 
 def str_decode_utf_32(s, errors, final=True,
                       errorhandler=None):
-    result, lgt_ = str_decode_utf_32_helper(
+    result, lgt = str_decode_utf_32_helper(
         s, errors, final, errorhandler, "native", 'utf-32-' + BYTEORDER2,
         allow_surrogates=False)
     return result, lgt
@@ -1251,10 +1249,10 @@
 
 def str_decode_utf_32_le(s, errors, final=True,
                          errorhandler=None):
-    result, lgt_ = str_decode_utf_32_helper(
+    result, lgt = str_decode_utf_32_helper(
         s, errors, final, errorhandler, "little", 'utf-32-le',
         allow_surrogates=False)
-    return result, c, lgt
+    return result, lgt
 
 BOM32_DIRECT  = intmask(0x0000FEFF)
 BOM32_REVERSE = intmask(0xFFFE0000)
@@ -1362,11 +1360,79 @@
         result.append(c2)
         result.append(c3)
 
+def utf8_encode_utf_32_helper(s, errors,
+                                 errorhandler=None,
+                                 allow_surrogates=True,
+                                 byteorder='little',
+                                 public_encoding_name='utf32'):
+    # s is utf8
+    size = len(s)
+    if size == 0:
+        if byteorder == 'native':
+            result = StringBuilder(4)
+            _STORECHAR32(result, 0xFEFF, BYTEORDER)
+            return result.build()
+        return ""
+
+    result = StringBuilder(size * 4 + 4)
+    if byteorder == 'native':
+        _STORECHAR32(result, 0xFEFF, BYTEORDER)
+        byteorder = BYTEORDER
+
+    pos = 0
+    index = 0
+    while pos < size:
+        try:
+            ch = rutf8.codepoint_at_pos(s, pos)
+            pos = rutf8.next_codepoint_pos(s, pos)
+        except IndexError:
+            # malformed codepoint, blindly use ch
+            ch = ord(s[pos])
+            pos += 1
+            if errorhandler:
+                res_8, newindex = errorhandler(
+                    errors, public_encoding_name, 'malformed unicode',
+                    s, pos - 1, pos)
+                if res_8:
+                    for cp in rutf8.Utf8StringIterator(res_8):
+                        if cp < 0xD800:
+                            _STORECHAR32(result, cp, byteorder)
+                        else:
+                            errorhandler('strict', public_encoding_name,
+                                     'malformed unicode',
+                                 s, pos-1, pos)
+                else:
+                    _STORECHAR32(result, ch, byteorder)
+            else:
+                _STORECHAR32(result, ch, byteorder)
+            index += 1
+            continue
+        if not allow_surrogates and 0xD800 <= ch < 0xE000:
+            res_8, newindex = errorhandler(
+                errors, public_encoding_name, 'surrogates not allowed',
+                s, pos - 1, pos)
+            for ch in rutf8.Utf8StringIterator(res_8):
+                if ch < 0xD800:
+                    _STORECHAR32(result, ch, byteorder)
+                else:
+                    errorhandler(
+                        'strict', public_encoding_name, 'surrogates not allowed',
+                        s, pos - 1, pos)
+            if index != newindex:  # Should be uncommon
+                index = newindex
+                pos = rutf8._pos_at_index(s, newindex)
+            continue
+        _STORECHAR32(result, ch, byteorder)
+        index += 1
+
+    return result.build()
+
 def unicode_encode_utf_32_helper(s, errors,
                                  errorhandler=None,
                                  allow_surrogates=True,
                                  byteorder='little',
                                  public_encoding_name='utf32'):
+    # s is uunicode
     size = len(s)
     if size == 0:
         if byteorder == 'native':
@@ -1430,19 +1496,19 @@
 
 def utf8_encode_utf_32(s, errors,
                           errorhandler=None, allow_surrogates=True):
-    return unicode_encode_utf_32_helper(s.decode('utf8'), errors, errorhandler,
+    return utf8_encode_utf_32_helper(s, errors, errorhandler,
                                         allow_surrogates, "native",
                                         'utf-32-' + BYTEORDER2)
 
 def utf8_encode_utf_32_be(s, errors,
                                   errorhandler=None, allow_surrogates=True):
-    return unicode_encode_utf_32_helper(s.decode('utf8'), errors, errorhandler,
+    return utf8_encode_utf_32_helper(s, errors, errorhandler,
                                         allow_surrogates, "big",
                                         'utf-32-be')
 
 def utf8_encode_utf_32_le(s, errors,
                                   errorhandler=None, allow_surrogates=True):
-    return unicode_encode_utf_32_helper(s.decode('utf8'), errors, errorhandler,
+    return utf8_encode_utf_32_helper(s, errors, errorhandler,
                                         allow_surrogates, "little",
                                         'utf-32-le')
 # ____________________________________________________________
@@ -1548,14 +1614,13 @@
         result.append(c)
         pos += 1
     r = result.build()
-    lgt = rutf8.check_utf8(r, True)
+    lgt = rutf8.codepoints_in_utf8(r)
     return r, lgt
 
 def utf8_encode_charmap(s, errors, errorhandler=None, mapping=None):
-    size = len(s)
     if mapping is None:
         return utf8_encode_latin_1(s, errors, errorhandler=errorhandler)
-
+    size = len(s)
     if size == 0:
         return ''
     result = StringBuilder(size)
diff --git a/pypy/module/_codecs/interp_codecs.py b/pypy/module/_codecs/interp_codecs.py
--- a/pypy/module/_codecs/interp_codecs.py
+++ b/pypy/module/_codecs/interp_codecs.py
@@ -49,8 +49,8 @@
                 length = len(input)
             else:
                 w_cls = space.w_UnicodeEncodeError
-                length = rutf8.check_utf8(input, allow_surrogates=True)
-                w_input = space.newutf8(input, length)
+                length = rutf8.codepoints_in_utf8(input)
+                w_input = space.newtext((input, length))
             w_exc =  space.call_function(
                 w_cls,
                 space.newtext(encoding),
@@ -89,8 +89,7 @@
                 raise oefmt(space.w_IndexError,
                             "position %d from error handler out of bounds",
                             newpos)
-            w_replace = space.convert_to_w_unicode(w_replace)
-            return w_replace._utf8, newpos
+            return space.utf8_w(w_replace), newpos
         return call_errorhandler
 
     def make_decode_errorhandler(self, space):
@@ -238,7 +237,6 @@
                     "don't know how to handle %T in error callback", w_exc)
 
 def xmlcharrefreplace_errors(space, w_exc):
-    from pypy.interpreter import unicodehelper
 
     check_exception(space, w_exc)
     if space.isinstance_w(w_exc, space.w_UnicodeEncodeError):
@@ -267,8 +265,8 @@
                     "don't know how to handle %T in error callback", w_exc)
 
 def backslashreplace_errors(space, w_exc):
-    from pypy.interpreter import unicodehelper
 
+    import pdb;pdb.set_trace()
     check_exception(space, w_exc)
     if (space.isinstance_w(w_exc, space.w_UnicodeEncodeError) or
         space.isinstance_w(w_exc, space.w_UnicodeTranslateError)):
@@ -630,34 +628,14 @@
     func = _find_implementation(rname)
     @unwrap_spec(errors='text_or_none')
     def wrap_encoder(space, w_arg, errors="strict"):
-        from pypy.interpreter import unicodehelper
-
+        # w_arg is a W_Unicode or W_Bytes?
         w_arg = unicodehelper.convert_arg_to_w_unicode(space, w_arg, rname)
         if errors is None:
             errors = 'strict'
         state = space.fromcache(CodecState)
-        utf8len = w_arg._length
-        # XXX deal with func() returning length or not
+        ulen = w_arg._length
         result = func(w_arg._utf8, errors, state.encode_error_handler)
-        return space.newtuple([space.newbytes(result.encode('utf8')), space.newint(utf8len)])
-    wrap_encoder.__name__ = func.__name__
-    globals()[name] = wrap_encoder
-
-def make_utf_encoder_wrapper(name):
-    rname = "unicode_encode_%s" % (name.replace("_encode", ""), )
-    func = _find_implementation(rname)
-    @unwrap_spec(errors='text_or_none')
-    def wrap_encoder(space, w_arg, errors="strict"):
-        from pypy.interpreter import unicodehelper
-
-        w_arg = unicodehelper.convert_arg_to_w_unicode(space, w_arg, rname)
-        if errors is None:
-            errors = 'strict'
-        state = space.fromcache(CodecState)
-        utf8len = w_arg._length
-        result = func(w_arg._utf8, errors, state.encode_error_handler,
-                      allow_surrogates=False)
-        return space.newtuple([space.newbytes(result), space.newint(utf8len)])
+        return space.newtuple([space.newbytes(result), space.newint(ulen)])
     wrap_encoder.__name__ = func.__name__
     globals()[name] = wrap_encoder
 
@@ -667,16 +645,13 @@
     @unwrap_spec(string='bufferstr', errors='text_or_none',
                  w_final=WrappedDefault(False))
     def wrap_decoder(space, string, errors="strict", w_final=None):
-        from pypy.interpreter import unicodehelper
 
         if errors is None:
             errors = 'strict'
         final = space.is_true(w_final)
         state = space.fromcache(CodecState)
-        result, consumed, length = func(string, errors,
-                                              final, state.decode_error_handler)
-        return space.newtuple([space.newutf8(result, length),
-                               space.newint(consumed)])
+        result, length = func(string, errors, final, state.decode_error_handler)
+        return space.newtuple([space.newutf8(result, length), space.newint(length)])
     wrap_decoder.__name__ = func.__name__
     globals()[name] = wrap_decoder
 
@@ -730,11 +705,11 @@
             errors = 'strict'
         final = space.is_true(w_final)
         state = space.fromcache(CodecState)
-        result, consumed = runicode.str_decode_mbcs(
+        result, length = runicode.str_decode_mbcs(
             string, len(string), errors,
             final, state.decode_error_handler,
             force_ignore=False)
-        return space.newtuple([space.newtext(result), space.newint(consumed)])
+        return space.newtuple([space.newtext(result, length), space.newint(length)])
 
 # utf-8 functions are not regular, because we have to pass
 # "allow_surrogates=False"
@@ -755,7 +730,6 @@
 @unwrap_spec(string='bufferstr', errors='text_or_none',
              w_final = WrappedDefault(False))
 def utf_8_decode(space, string, errors="strict", w_final=None):
-    from pypy.interpreter import unicodehelper
 
     if errors is None:
         errors = 'strict'
@@ -765,10 +739,10 @@
     try:
         lgt = rutf8.check_utf8(string, allow_surrogates=True)
     except rutf8.CheckError:
-        res, consumed, lgt = unicodehelper.str_decode_utf8(string,
+        res, lgt = unicodehelper.str_decode_utf8(string,
             errors, final, state.decode_error_handler)
         return space.newtuple([space.newutf8(res, lgt),
-                               space.newint(consumed)])
+                               space.newint(lgt)])
     else:
         return space.newtuple([space.newutf8(string, lgt),
                                space.newint(len(string))])
@@ -788,15 +762,11 @@
         byteorder = 'little'
     else:
         byteorder = 'big'
-    consumed = len(data)
-    if final:
-        consumed = 0
-    res, consumed, lgt, byteorder = str_decode_utf_16_helper(
+    res, lgt = str_decode_utf_16_helper(
         data, errors, final,
         state.decode_error_handler, byteorder)
     return space.newtuple([space.newutf8(res, lgt),
-                           space.newint(consumed),
-                           space.newint(byteorder)])
+                           space.newint(lgt)])
 
 @unwrap_spec(data='bufferstr', errors='text_or_none', byteorder=int,
              w_final=WrappedDefault(False))
@@ -811,15 +781,11 @@
         byteorder = 'little'
     else:
         byteorder = 'big'
-    consumed = len(data)
-    if final:
-        consumed = 0
-    res, consumed, lgt, byteorder = str_decode_utf_32_helper(
+    res, lgt = str_decode_utf_32_helper(
         data, errors, final,
         state.decode_error_handler, byteorder)
     return space.newtuple([space.newutf8(res, lgt),
-                           space.newint(consumed),
-                           space.newint(byteorder)])
+                           space.newint(lgt)])
 
 # ____________________________________________________________
 # Charmap
@@ -902,7 +868,6 @@
 
 @unwrap_spec(string='bufferstr', errors='text_or_none')
 def charmap_decode(space, string, errors="strict", w_mapping=None):
-    from pypy.interpreter import unicodehelper
 
     if errors is None:
         errors = 'strict'
@@ -917,14 +882,13 @@
 
     final = True
     state = space.fromcache(CodecState)
-    result, consumed, lgt = unicodehelper.str_decode_charmap(
+    result, lgt = unicodehelper.str_decode_charmap(
         string, errors, final, state.decode_error_handler, mapping)
     return space.newtuple([space.newutf8(result, lgt),
-                           space.newint(consumed)])
+                           space.newint(len(string))])
 
 @unwrap_spec(errors='text_or_none')
 def charmap_encode(space, w_unicode, errors="strict", w_mapping=None):
-    from pypy.interpreter import unicodehelper
 
     if errors is None:
         errors = 'strict'
@@ -974,7 +938,6 @@
 @unwrap_spec(errors='text_or_none', w_final=WrappedDefault(False))
 def unicode_escape_decode(space, w_string, errors="strict", w_final=None):
     string = space.getarg_w('s*', w_string).as_str()
-    from pypy.interpreter import unicodehelper
 
     if errors is None:
         errors = 'strict'
@@ -983,12 +946,13 @@
 
     unicode_name_handler = state.get_unicodedata_handler(space)
 
-    result, consumed, lgt = unicodehelper.str_decode_unicode_escape(
+    result, lgt = unicodehelper.str_decode_unicode_escape(
         string, errors,
         final, state.decode_error_handler,
         unicode_name_handler)
 
-    return space.newtuple([space.newutf8(result, lgt), space.newint(consumed)])
+    s_len = len(string)
+    return space.newtuple([space.newutf8(result, lgt), space.newint(s_len)])
 
 # ____________________________________________________________
 # Raw Unicode escape (accepts bytes or str)
@@ -1000,17 +964,16 @@
         errors = 'strict'
     final = space.is_true(w_final)
     state = space.fromcache(CodecState)
-    result, consumed = runicode.str_decode_raw_unicode_escape(
+    result, lgt = runicode.str_decode_raw_unicode_escape(
         string, len(string), errors,
         final, state.decode_error_handler)
-    return space.newtuple([space.newtext(result), space.newint(consumed)])
+    return space.newtuple([space.newtext(result), space.newint(lgt)])
 
 # ____________________________________________________________
 # Unicode-internal
 
 @unwrap_spec(errors='text_or_none')
 def unicode_internal_decode(space, w_string, errors="strict"):
-    from pypy.interpreter import unicodehelper
 
     if errors is None:
         errors = 'strict'
@@ -1028,11 +991,11 @@
 
     final = True
     state = space.fromcache(CodecState)
-    result, consumed, lgt = unicodehelper.str_decode_unicode_internal(
+    result, lgt = unicodehelper.str_decode_unicode_internal(
         string, errors,
         final, state.decode_error_handler)
     return space.newtuple([space.newutf8(result, lgt),
-                           space.newint(consumed)])
+                           space.newint(lgt)])
 
 @unwrap_spec(errors='text_or_none')
 def unicode_internal_encode(space, w_uni, errors="strict"):
@@ -1041,11 +1004,12 @@
     if errors is None:
         errors = 'strict'
     if space.isinstance_w(w_uni, space.w_unicode):
-        uni = space.utf8_w(w_uni)
+        utf8 = space.utf8_w(w_uni)
         state = space.fromcache(CodecState)
-        result = runicode.unicode_encode_unicode_internal(
-            uni, len(uni), errors, state.encode_error_handler)
-        return space.newtuple([space.newbytes(result), space.newint(len(uni))])
+        result = unicodehelper.utf8_encode_unicode_internal(
+            utf8, errors, state.encode_error_handler)
+        w_lgt = space.newint(space.len_w(w_uni))
+        return space.newtuple([space.newbytes(result), w_lgt])
     else:
         # special case for this codec: bytes are returned as is
         string = space.charbuf_w(w_uni)
diff --git a/pypy/module/_codecs/test/test_codecs.py b/pypy/module/_codecs/test/test_codecs.py
--- a/pypy/module/_codecs/test/test_codecs.py
+++ b/pypy/module/_codecs/test/test_codecs.py
@@ -107,7 +107,8 @@
         import sys
         assert charmap_decode(b'', 'strict', 'blablabla') == ('', 0)
         assert charmap_decode(b'xxx') == ('xxx', 3)
-        assert charmap_decode(b'xxx', 'strict', {ord('x'): 'XX'}) == ('XXXXXX', 3)
+        res = charmap_decode(b'xxx', 'strict', {ord('x'): 'XX'})
+        assert  res == ('XXXXXX', 3)
         map = tuple([chr(i) for i in range(256)])
         assert charmap_decode(b'xxx\xff', 'strict', map) == ('xxx\xff', 4)
 
@@ -123,7 +124,11 @@
 
     def test_escape_decode(self):
         from _codecs import unicode_escape_decode as decode
-        assert decode('\\\x80') == (u'\\\x80', 2)
+        import sys
+        if sys.version_info[0] < 3:
+            assert decode('\\\x80') == (u'\\\x80', 2)
+        else:
+            assert decode('\\\x80') == (u'\\\xc2\x80', 3)
 
     def test_escape_decode_errors(self):
         from _codecs import escape_decode as decode
@@ -137,10 +142,15 @@
         assert decode(br"[\x0]\x0", "replace") == (b"[?]?", 8)
 
     def test_unicode_escape(self):
+        import sys
         from _codecs import unicode_escape_encode, unicode_escape_decode
         assert unicode_escape_encode('abc') == ('abc'.encode('unicode_escape'), 3)
         assert unicode_escape_decode(b'abc') == (b'abc'.decode('unicode_escape'), 3)
-        assert unicode_escape_decode(b'\\x61\\x62\\x63') == ('abc', 12)
+        if sys.version_info[0] < 3:
+            lgt = 12
+        else:
+            lgt = 3
+        assert unicode_escape_decode(b'\\x61\\x62\\x63') == ('abc', lgt)
 
 
 class AppTestPartialEvaluation:
@@ -338,23 +348,26 @@
 
     def test_unicode_escape_decode_errors(self):
         from _codecs import unicode_escape_decode, raw_unicode_escape_decode
+        import sys
         for decode in [unicode_escape_decode, raw_unicode_escape_decode]:
             for c, d in ('u', 4), ('U', 4):
                 for i in range(d):
                     raises(UnicodeDecodeError, decode, "\\" + c + "0"*i)
                     raises(UnicodeDecodeError, decode, "[\\" + c + "0"*i + "]")
                     data = "[\\" + c + "0"*i + "]\\" + c + "0"*i
-                    assert decode(data, "ignore") == (u"[]", len(data))
-                    assert decode(data, "replace") == (u"[\ufffd]\ufffd", len(data))
+                    lgt = len(data)
+                    assert decode(data, "ignore") == (u"[]", lgt)
+                    assert decode(data, "replace") == (u"[\ufffd]\ufffd", lgt)
             raises(UnicodeDecodeError, decode, r"\U00110000")
-            assert decode(r"\U00110000", "ignore") == (u"", 10)
-            assert decode(r"\U00110000", "replace") == (u"\ufffd", 10)
+            lgt = 10
+            assert decode(r"\U00110000", "ignore") == (u"", lgt)
+            assert decode(r"\U00110000", "replace") == (u"\ufffd", lgt)
         exc = raises(UnicodeDecodeError, unicode_escape_decode, b"\u1z32z3", 'strict')
         assert str(exc.value) == r"'unicodeescape' codec can't decode bytes in position 0-2: truncated \uXXXX escape"
         exc = raises(UnicodeDecodeError, raw_unicode_escape_decode, b"\u1z32z3", 'strict')
-        assert str(exc.value) == r"'rawunicodeescape' codec can't decode bytes in position 0-2: truncated \uXXXX"
+        assert str(exc.value) == r"'rawunicodeescape' codec can't decode bytes in position 0-2: truncated \uXXXX escape"
         exc = raises(UnicodeDecodeError, raw_unicode_escape_decode, b"\U1z32z3", 'strict')
-        assert str(exc.value) == r"'rawunicodeescape' codec can't decode bytes in position 0-2: truncated \uXXXX"
+        assert str(exc.value) == r"'rawunicodeescape' codec can't decode bytes in position 0-2: truncated \UXXXXXXXX escape"
 
     def test_escape_encode(self):
         import _codecs
@@ -920,11 +933,11 @@
         assert _codecs.utf_16_be_decode(b) == (u'', 0)
         assert _codecs.utf_16_decode(b) == (u'', 0)
         assert _codecs.utf_16_le_decode(b) == (u'', 0)
-        assert _codecs.utf_16_ex_decode(b) == (u'', 0, 0)
+        assert _codecs.utf_16_ex_decode(b) == (u'', 0)
         assert _codecs.utf_32_decode(b) == (u'', 0)
         assert _codecs.utf_32_be_decode(b) == (u'', 0)
         assert _codecs.utf_32_le_decode(b) == (u'', 0)
-        assert _codecs.utf_32_ex_decode(b) == (u'', 0, 0)
+        assert _codecs.utf_32_ex_decode(b) == (u'', 0)
         assert _codecs.charmap_decode(b) == (u'', 0)
         assert _codecs.unicode_escape_decode(b) == (u'', 0)
         assert _codecs.raw_unicode_escape_decode(b) == (u'', 0)
diff --git a/pypy/module/_codecs/test/test_locale.py b/pypy/module/_codecs/test/test_locale.py
--- a/pypy/module/_codecs/test/test_locale.py
+++ b/pypy/module/_codecs/test/test_locale.py
@@ -4,7 +4,8 @@
 from pypy.module._codecs.locale import (
     str_decode_locale_surrogateescape,
     unicode_encode_locale_surrogateescape)
-from rpython.rlib import rlocale, runicode
+from rpython.rlib import rlocale
+from pypy.interpreter import unicodehelper
 
 class TestLocaleCodec(object):
 
@@ -18,11 +19,11 @@
             rlocale.setlocale(rlocale.LC_ALL, cls.oldlocale)
 
     def getdecoder(self, encoding):
-        return getattr(runicode, "str_decode_%s" % encoding.replace("-", "_"))
+        return getattr(unicodehelper, "str_decode_%s" % encoding.replace("-", ""))
 
     def getencoder(self, encoding):
-        return getattr(runicode,
-                       "unicode_encode_%s" % encoding.replace("-", "_"))
+        return getattr(unicodehelper,
+                       "utf8_encode_%s" % encoding.replace("-", "_"))
 
     def getstate(self):
         return self.space.fromcache(interp_codecs.CodecState)
@@ -39,8 +40,8 @@
         locale_encoder = unicode_encode_locale_surrogateescape
         utf8_encoder = self.getencoder('utf-8')
         for val in u'foo', u' &#26085;&#26412;', u'\U0001320C':
-            assert (locale_encoder(val) ==
-                    utf8_encoder(val, len(val), None))
+            assert (locale_encoder(val).encode('utf8') ==
+                    utf8_encoder(val, 'strict', True, None))
 
     def test_encode_locale_errorhandler(self):
         self.setlocale("en_US.UTF-8")
@@ -48,17 +49,17 @@
         utf8_encoder = self.getencoder('utf-8')
         encode_error_handler = self.getstate().encode_error_handler
         for val in u'foo\udc80bar', u'\udcff\U0001320C':
-            expected = utf8_encoder(val, len(val), 'surrogateescape',
+            expected = utf8_encoder(val, 'surrogateescape',
                                     encode_error_handler)
-            assert locale_encoder(val) == expected
+            assert locale_encoder(val).encode('utf8') == expected
 
     def test_decode_locale(self):
         self.setlocale("en_US.UTF-8")
         locale_decoder = str_decode_locale_surrogateescape
         utf8_decoder = self.getdecoder('utf-8')
         for val in 'foo', ' \xe6\x97\xa5\xe6\x9c\xac', '\xf0\x93\x88\x8c':
-            assert (locale_decoder(val) ==
-                    utf8_decoder(val, len(val), None)[0])
+            assert (locale_decoder(val).encode('utf8') ==
+                    utf8_decoder(val, 'strict', True, None)[0])
 
     def test_decode_locale_errorhandler(self):
         self.setlocale("en_US.UTF-8")
@@ -66,6 +67,6 @@
         utf8_decoder = self.getdecoder('utf-8')
         decode_error_handler = self.getstate().decode_error_handler
         val = 'foo\xe3bar'
-        expected = utf8_decoder(val, len(val), 'surrogateescape', True,
+        expected = utf8_decoder(val, 'surrogateescape', True,
                                 decode_error_handler)[0]
-        assert locale_decoder(val) == expected
+        assert locale_decoder(val).encode('utf8') == expected
diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py
--- a/pypy/objspace/std/objspace.py
+++ b/pypy/objspace/std/objspace.py
@@ -399,7 +399,7 @@
         return self.newtext(s)
 
     def newutf8(self, utf8s, length):
-        assert utf8s is not None
+        assert isinstance(utf8s, str)
         return W_UnicodeObject(utf8s, length)
 
     def newfilename(self, s):
diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -43,7 +43,7 @@
         # XXX checking, remove before any performance measurments
         #     ifdef not_running_in_benchmark
         if not we_are_translated():
-            lgt = rutf8.check_utf8(utf8str, True)
+            lgt = rutf8.codepoints_in_utf8(utf8str)
             assert lgt == length
 
     @staticmethod
@@ -57,7 +57,7 @@
 
     def unwrap(self, space):
         # for testing
-        return self.realunicode_w(space)
+        return space.realunicode_w(self)
 
     def is_w(self, space, w_other):
         if not isinstance(w_other, W_UnicodeObject):
@@ -95,9 +95,6 @@
     def utf8_w(self, space):
         return self._utf8
 
-    def realunicode_w(self, space):
-        return self._utf8.decode('utf8')
-
     def listview_utf8(self):
         assert self.is_ascii()
         return _create_list_from_unicode(self._utf8)
@@ -1190,9 +1187,12 @@
     utf8 = space.utf8_w(w_object)
     idx = rutf8.surrogate_in_utf8(utf8)
     if idx >= 0:
-        eh = unicodehelper.encode_error_handler(space)
-        eh(None, "utf8", "surrogates not allowed", utf8,
-            idx, idx + 1)
+        print 'surrogate in unicodeobject.encode_object(', w_object, ',', encoding, ',', errors, ')', 'raising'
+        if errors is None:
+            w_err_handler = unicodehelper.encode_error_handler(space)
+        else:
+            w_err_handler = lookup_error(space, errors)
+        w_err_handler(None, "utf8", "surrogates not allowed", utf8, idx, idx + 1)
     if errors is None or errors == 'strict':
         if encoding is None or encoding == 'utf-8':
             #if rutf8.has_surrogates(utf8):
diff --git a/rpython/rlib/runicode.py b/rpython/rlib/runicode.py
--- a/rpython/rlib/runicode.py
+++ b/rpython/rlib/runicode.py
@@ -1401,7 +1401,7 @@
             endinpos += 1
         res, pos = errorhandler(errors, encoding,
                                 message, s, pos-2, endinpos)
-        builder.append(res)
+        builder.append(res.decode('utf8'))
     else:
         try:
             chr = r_uint(int(s[pos:pos+digits], 16))
@@ -1411,7 +1411,7 @@
                 endinpos += 1
             res, pos = errorhandler(errors, encoding,
                                     message, s, pos-2, endinpos)
-            builder.append(res)
+            builder.append(res.decode('utf8'))
         else:
             # when we get here, chr is a 32-bit unicode character
             if chr <= MAXUNICODE:
@@ -1427,7 +1427,7 @@
                 message = "illegal Unicode character"
                 res, pos = errorhandler(errors, encoding,
                                         message, s, pos-2, pos+digits)
-                builder.append(res)
+                builder.append(res.decode('utf8'))
     return pos
 
 def str_decode_unicode_escape(s, size, errors, final=False,
@@ -1708,8 +1708,12 @@
             pos += 1
             continue
 
-        digits = 4 if s[pos] == 'u' else 8
-        message = "truncated \\uXXXX"
+        if s[pos] == 'u':
+            digits = 4
+            message = "truncated \\uXXXX escape"
+        else:
+            digits = 8
+            message = "truncated \\UXXXXXXXX escape"
         pos += 1
         pos = hexescape(result, s, pos, digits,
                         "rawunicodeescape", errorhandler, message, errors)

From pypy.commits at gmail.com  Fri Jul 27 21:01:33 2018
From: pypy.commits at gmail.com (rlamy)
Date: Fri, 27 Jul 2018 18:01:33 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6: hg merge py3.5
Message-ID: <5b5bc06d.1c69fb81.0b47.cabd@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.6
Changeset: r94909:ebe135031f7e
Date: 2018-07-28 02:00 +0100
http://bitbucket.org/pypy/pypy/changeset/ebe135031f7e/

Log:	hg merge py3.5

diff too long, truncating to 2000 out of 4128 lines

diff --git a/lib-python/3/fractions.py b/lib-python/3/fractions.py
--- a/lib-python/3/fractions.py
+++ b/lib-python/3/fractions.py
@@ -542,7 +542,7 @@
         else:
             return Fraction(round(self / shift) * shift)
 
-    def __hash__(self):
+    def __hash__(self, CACHE=[-1] * 1001):
         """hash(self)"""
 
         # XXX since this method is expensive, consider caching the result
@@ -556,12 +556,23 @@
         # dinv is the inverse of self._denominator modulo the prime
         # _PyHASH_MODULUS, or 0 if self._denominator is divisible by
         # _PyHASH_MODULUS.
-        dinv = pow(self._denominator, _PyHASH_MODULUS - 2, _PyHASH_MODULUS)
+
+        # PyPy3: added caching of pow() results for denominators up to 1000
+        denom = self._denominator
+        assert denom >= 0
+        if denom < len(CACHE):
+            dinv = CACHE[denom]
+            if dinv == -1:
+                dinv = pow(denom, _PyHASH_MODULUS - 2, _PyHASH_MODULUS)
+                CACHE[denom] = dinv
+        else:
+            dinv = pow(denom, _PyHASH_MODULUS - 2, _PyHASH_MODULUS)
+
         if not dinv:
             hash_ = _PyHASH_INF
         else:
             hash_ = abs(self._numerator) * dinv % _PyHASH_MODULUS
-        result = hash_ if self >= 0 else -hash_
+        result = hash_ if self._numerator >= 0 else -hash_
         return -2 if result == -1 else result
 
     def __eq__(a, b):
diff --git a/lib-python/3/hashlib.py b/lib-python/3/hashlib.py
--- a/lib-python/3/hashlib.py
+++ b/lib-python/3/hashlib.py
@@ -162,9 +162,14 @@
     __get_hash = __get_openssl_constructor
     algorithms_available = algorithms_available.union(
             _hashlib.openssl_md_meth_names)
-except ImportError:
+except ImportError as e:
     new = __py_new
     __get_hash = __get_builtin_constructor
+    # added by PyPy
+    import warnings
+    warnings.warn("The _hashlib module is not available, falling back "
+                  "to a much slower implementation (%s)" % str(e),
+                  RuntimeWarning)
 
 try:
     # OpenSSL's PKCS5_PBKDF2_HMAC requires OpenSSL 1.0+ with HMAC and SHA
diff --git a/lib_pypy/_winapi.py b/lib_pypy/_winapi.py
--- a/lib_pypy/_winapi.py
+++ b/lib_pypy/_winapi.py
@@ -98,9 +98,11 @@
     def GetOverlappedResult(self, wait):
         transferred = _ffi.new('DWORD[1]', [0])
         res = _kernel32.GetOverlappedResult(self.handle, self.overlapped, transferred, wait != 0)
-        if not res:
-            res = GetLastError()
-        if res in (ERROR_SUCCESS, ERROR_MORE_DATA, ERROR_OPERATION_ABORTED):
+        if res:
+            err = ERROR_SUCCESS
+        else:
+            err = GetLastError()
+        if err in (ERROR_SUCCESS, ERROR_MORE_DATA, ERROR_OPERATION_ABORTED):
             self.completed = 1
             self.pending = 0
         elif res == ERROR_IO_INCOMPLETE:
@@ -133,7 +135,7 @@
         assert success == 0
         err = _kernel32.GetLastError()
         if err == ERROR_IO_PENDING:
-            overlapped[0].pending = 1
+            ov.pending = 1
         elif err == ERROR_PIPE_CONNECTED:
             _kernel32.SetEvent(ov.overlapped[0].hEvent)
         else:
diff --git a/lib_pypy/cffi.egg-info/PKG-INFO b/lib_pypy/cffi.egg-info/PKG-INFO
--- a/lib_pypy/cffi.egg-info/PKG-INFO
+++ b/lib_pypy/cffi.egg-info/PKG-INFO
@@ -1,6 +1,6 @@
 Metadata-Version: 1.1
 Name: cffi
-Version: 1.11.5
+Version: 1.12.0
 Summary: Foreign Function Interface for Python calling C code.
 Home-page: http://cffi.readthedocs.org
 Author: Armin Rigo, Maciej Fijalkowski
diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py
--- a/lib_pypy/cffi/__init__.py
+++ b/lib_pypy/cffi/__init__.py
@@ -4,8 +4,8 @@
 from .api import FFI
 from .error import CDefError, FFIError, VerificationError, VerificationMissing
 
-__version__ = "1.11.5"
-__version_info__ = (1, 11, 5)
+__version__ = "1.12.0"
+__version_info__ = (1, 12, 0)
 
 # The verifier module file names are based on the CRC32 of a string that
 # contains the following version number.  It may be older than __version__
diff --git a/lib_pypy/cffi/_embedding.h b/lib_pypy/cffi/_embedding.h
--- a/lib_pypy/cffi/_embedding.h
+++ b/lib_pypy/cffi/_embedding.h
@@ -221,7 +221,7 @@
 
         if (f != NULL && f != Py_None) {
             PyFile_WriteString("\nFrom: " _CFFI_MODULE_NAME
-                               "\ncompiled with cffi version: 1.11.5"
+                               "\ncompiled with cffi version: 1.12.0"
                                "\n_cffi_backend module: ", f);
             modules = PyImport_GetModuleDict();
             mod = PyDict_GetItemString(modules, "_cffi_backend");
diff --git a/lib_pypy/cffi/api.py b/lib_pypy/cffi/api.py
--- a/lib_pypy/cffi/api.py
+++ b/lib_pypy/cffi/api.py
@@ -96,18 +96,21 @@
             self.CData, self.CType = backend._get_types()
         self.buffer = backend.buffer
 
-    def cdef(self, csource, override=False, packed=False):
+    def cdef(self, csource, override=False, packed=False, pack=None):
         """Parse the given C source.  This registers all declared functions,
         types, and global variables.  The functions and global variables can
         then be accessed via either 'ffi.dlopen()' or 'ffi.verify()'.
         The types can be used in 'ffi.new()' and other functions.
         If 'packed' is specified as True, all structs declared inside this
         cdef are packed, i.e. laid out without any field alignment at all.
+        Alternatively, 'pack' can be a small integer, and requests for
+        alignment greater than that are ignored (pack=1 is equivalent to
+        packed=True).
         """
-        self._cdef(csource, override=override, packed=packed)
+        self._cdef(csource, override=override, packed=packed, pack=pack)
 
-    def embedding_api(self, csource, packed=False):
-        self._cdef(csource, packed=packed, dllexport=True)
+    def embedding_api(self, csource, packed=False, pack=None):
+        self._cdef(csource, packed=packed, pack=pack, dllexport=True)
         if self._embedding is None:
             self._embedding = ''
 
diff --git a/lib_pypy/cffi/backend_ctypes.py b/lib_pypy/cffi/backend_ctypes.py
--- a/lib_pypy/cffi/backend_ctypes.py
+++ b/lib_pypy/cffi/backend_ctypes.py
@@ -730,7 +730,8 @@
         return self._new_struct_or_union('union', name, ctypes.Union)
 
     def complete_struct_or_union(self, CTypesStructOrUnion, fields, tp,
-                                 totalsize=-1, totalalignment=-1, sflags=0):
+                                 totalsize=-1, totalalignment=-1, sflags=0,
+                                 pack=0):
         if totalsize >= 0 or totalalignment >= 0:
             raise NotImplementedError("the ctypes backend of CFFI does not support "
                                       "structures completed by verify(); please "
@@ -751,6 +752,8 @@
                 bfield_types[fname] = Ellipsis
         if sflags & 8:
             struct_or_union._pack_ = 1
+        elif pack:
+            struct_or_union._pack_ = pack
         struct_or_union._fields_ = cfields
         CTypesStructOrUnion._bfield_types = bfield_types
         #
diff --git a/lib_pypy/cffi/cparser.py b/lib_pypy/cffi/cparser.py
--- a/lib_pypy/cffi/cparser.py
+++ b/lib_pypy/cffi/cparser.py
@@ -306,11 +306,25 @@
             msg = 'parse error\n%s' % (msg,)
         raise CDefError(msg)
 
-    def parse(self, csource, override=False, packed=False, dllexport=False):
+    def parse(self, csource, override=False, packed=False, pack=None,
+                    dllexport=False):
+        if packed:
+            if packed != True:
+                raise ValueError("'packed' should be False or True; use "
+                                 "'pack' to give another value")
+            if pack:
+                raise ValueError("cannot give both 'pack' and 'packed'")
+            pack = 1
+        elif pack:
+            if pack & (pack - 1):
+                raise ValueError("'pack' must be a power of two, not %r" %
+                    (pack,))
+        else:
+            pack = 0
         prev_options = self._options
         try:
             self._options = {'override': override,
-                             'packed': packed,
+                             'packed': pack,
                              'dllexport': dllexport}
             self._internal_parse(csource)
         finally:
diff --git a/lib_pypy/cffi/model.py b/lib_pypy/cffi/model.py
--- a/lib_pypy/cffi/model.py
+++ b/lib_pypy/cffi/model.py
@@ -342,7 +342,7 @@
     fixedlayout = None
     completed = 0
     partial = False
-    packed = False
+    packed = 0
 
     def __init__(self, name, fldnames, fldtypes, fldbitsize, fldquals=None):
         self.name = name
@@ -414,11 +414,14 @@
             fldtypes = [tp.get_cached_btype(ffi, finishlist)
                         for tp in self.fldtypes]
             lst = list(zip(self.fldnames, fldtypes, self.fldbitsize))
-            sflags = 0
+            extra_flags = ()
             if self.packed:
-                sflags = 8    # SF_PACKED
+                if self.packed == 1:
+                    extra_flags = (8,)    # SF_PACKED
+                else:
+                    extra_flags = (0, self.packed)
             ffi._backend.complete_struct_or_union(BType, lst, self,
-                                                  -1, -1, sflags)
+                                                  -1, -1, *extra_flags)
             #
         else:
             fldtypes = []
diff --git a/lib_pypy/cffi/recompiler.py b/lib_pypy/cffi/recompiler.py
--- a/lib_pypy/cffi/recompiler.py
+++ b/lib_pypy/cffi/recompiler.py
@@ -893,6 +893,12 @@
             else:
                 flags.append("_CFFI_F_CHECK_FIELDS")
             if tp.packed:
+                if tp.packed > 1:
+                    raise NotImplementedError(
+                        "%r is declared with 'pack=%r'; only 0 or 1 are "
+                        "supported in API mode (try to use \"...;\", which "
+                        "does not require a 'pack' declaration)" %
+                        (tp, tp.packed))
                 flags.append("_CFFI_F_PACKED")
         else:
             flags.append("_CFFI_F_EXTERNAL")
diff --git a/lib_pypy/pyrepl/simple_interact.py b/lib_pypy/pyrepl/simple_interact.py
--- a/lib_pypy/pyrepl/simple_interact.py
+++ b/lib_pypy/pyrepl/simple_interact.py
@@ -66,6 +66,10 @@
 
     while 1:
         try:
+            try:
+                sys.stdout.flush()
+            except:
+                pass
             ps1 = getattr(sys, 'ps1', '>>> ')
             ps2 = getattr(sys, 'ps2', '... ')
             try:
diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py
--- a/pypy/config/pypyoption.py
+++ b/pypy/config/pypyoption.py
@@ -43,14 +43,10 @@
     "_jitlog",
 ])
 
-from rpython.jit.backend import detect_cpu
-try:
-    if detect_cpu.autodetect().startswith('x86'):
-        if not sys.platform.startswith('openbsd'):
-            working_modules.add('_vmprof')
-            working_modules.add('faulthandler')
-except detect_cpu.ProcessorAutodetectError:
-    pass
+import rpython.rlib.rvmprof.cintf
+if rpython.rlib.rvmprof.cintf.IS_SUPPORTED:
+    working_modules.add('_vmprof')
+    working_modules.add('faulthandler')
 
 translation_modules = default_modules.copy()
 translation_modules.update([
@@ -323,3 +319,4 @@
     parser = to_optparse(config) #, useoptions=["translation.*"])
     option, args = parser.parse_args()
     print config
+    print working_modules
diff --git a/pypy/doc/config/objspace.disable_entrypoints.txt b/pypy/doc/config/objspace.disable_entrypoints.txt
new file mode 100644
diff --git a/pypy/doc/config/objspace.fstrings.txt b/pypy/doc/config/objspace.fstrings.txt
new file mode 100644
diff --git a/pypy/doc/config/objspace.hash.txt b/pypy/doc/config/objspace.hash.txt
new file mode 100644
diff --git a/pypy/doc/config/objspace.usemodules._frozen_importlib.txt b/pypy/doc/config/objspace.usemodules._frozen_importlib.txt
new file mode 100644
diff --git a/pypy/doc/config/objspace.usemodules._jitlog.txt b/pypy/doc/config/objspace.usemodules._jitlog.txt
new file mode 100644
diff --git a/pypy/doc/config/objspace.usemodules.faulthandler.txt b/pypy/doc/config/objspace.usemodules.faulthandler.txt
new file mode 100644
diff --git a/pypy/doc/config/translation.backendopt.replace_we_are_jitted.txt b/pypy/doc/config/translation.backendopt.replace_we_are_jitted.txt
new file mode 100644
diff --git a/pypy/doc/config/translation.jit_opencoder_model.txt b/pypy/doc/config/translation.jit_opencoder_model.txt
new file mode 100644
diff --git a/pypy/doc/config/translation.keepgoing.txt b/pypy/doc/config/translation.keepgoing.txt
new file mode 100644
diff --git a/pypy/doc/config/translation.libname.txt b/pypy/doc/config/translation.libname.txt
new file mode 100644
diff --git a/pypy/doc/config/translation.lto.txt b/pypy/doc/config/translation.lto.txt
new file mode 100644
diff --git a/pypy/doc/config/translation.profoptargs.txt b/pypy/doc/config/translation.profoptargs.txt
new file mode 100644
diff --git a/pypy/doc/config/translation.reverse_debugger.txt b/pypy/doc/config/translation.reverse_debugger.txt
new file mode 100644
diff --git a/pypy/doc/config/translation.split_gc_address_space.txt b/pypy/doc/config/translation.split_gc_address_space.txt
new file mode 100644
diff --git a/pypy/doc/tool/makecontributor.py b/pypy/doc/tool/makecontributor.py
--- a/pypy/doc/tool/makecontributor.py
+++ b/pypy/doc/tool/makecontributor.py
@@ -18,12 +18,13 @@
     'Antonio Cuni': ['antocuni', 'anto'],
     'Armin Rigo': ['arigo', 'arfigo', 'armin', 'arigato'],
     'Maciej Fijalkowski': ['fijal'],
-    'Carl Friedrich Bolz-Tereick': ['Carl Friedrich Bolz', 'cfbolz', 'cf'],
+    'Carl Friedrich Bolz-Tereick': ['Carl Friedrich Bolz', 'cfbolz', 'cf', 'cbolz'],
     'Samuele Pedroni': ['pedronis', 'samuele', 'samule'],
-    'Richard Plangger':['planrich'],
-    'Michael Hudson': ['mwh'],
+    'Richard Plangger': ['planrich', 'plan_rich'],
+    'Remi Meier': ['remi'],
+    'Michael Hudson-Doyle': ['mwh', 'Michael Hudson'],
     'Holger Krekel': ['hpk', 'holger krekel', 'holger', 'hufpk'],
-    "Amaury Forgeot d'Arc": ['afa'],
+    "Amaury Forgeot d'Arc": ['afa', 'amauryfa at gmail.com'],
     'Alex Gaynor': ['alex', 'agaynor'],
     'David Schneider': ['bivab', 'david'],
     'Christian Tismer': ['chris', 'christian', 'tismer',
@@ -41,7 +42,7 @@
     'Mark Pearse': ['mwp'],
     'Toon Verwaest': ['tverwaes'],
     'Eric van Riet Paap': ['ericvrp'],
-    'Jacob Hallen': ['jacob', 'jakob'],
+    'Jacob Hallen': ['jacob', 'jakob', 'jacob hallen'],
     'Anders Lehmann': ['ale', 'anders'],
     'Bert Freudenberg': ['bert'],
     'Boris Feigin': ['boris', 'boria'],
@@ -69,19 +70,25 @@
     'Manuel Jacob': ['mjacob'],
     'Rami Chowdhury': ['necaris'],
     'Stanislaw Halik': ['Stanislaw Halik', 'w31rd0'],
-    'Wenzhu Man':['wenzhu man', 'wenzhuman'],
-    'Anton Gulenko':['anton gulenko', 'anton_gulenko'],
-    'Richard Lancaster':['richardlancaster'],
-    'William Leslie':['William ML Leslie'],
-    'Spenser Bauman':['Spenser Andrew Bauman'],
-    'Raffael Tfirst':['raffael.tfirst at gmail.com'],
-    'timo':['timo at eistee.fritz.box'],
-    'Jasper Schulz':['Jasper.Schulz', 'jbs'],
-    'Aaron Gallagher':['"Aaron Gallagher'],
-    'Yasir Suhail':['yasirs'],
+    'Wenzhu Man': ['wenzhu man', 'wenzhuman'],
+    'Anton Gulenko': ['anton gulenko', 'anton_gulenko'],
+    'Richard Lancaster': ['richardlancaster'],
+    'William Leslie': ['William ML Leslie'],
+    'Spenser Bauman': ['Spenser Andrew Bauman'],
+    'Raffael Tfirst': ['raffael.tfirst at gmail.com'],
+    'timo': ['timo at eistee.fritz.box'],
+    'Jasper Schulz': ['Jasper.Schulz', 'jbs'],
+    'Aaron Gallagher': ['"Aaron Gallagher'],
+    'Yasir Suhail': ['yasirs'],
     'Squeaky': ['squeaky'],
-    "Amaury Forgeot d'Arc": ['amauryfa at gmail.com'],
     "Dodan Mihai": ['mihai.dodan at gmail.com'],
+    'Wim Lavrijsen': ['wlav'],
+    'Toon Verwaest': ['toon', 'tverwaes'],
+    'Seo Sanghyeon': ['sanxiyn'],
+    'Leonardo Santagada': ['santagada'],
+    'Laurence Tratt': ['ltratt'],
+    'Pieter Zieschang': ['pzieschang', 'p_zieschang at yahoo.de'],
+    'John Witulski': ['witulski'],
     }
 
 alias_map = {}
@@ -103,7 +110,8 @@
         return set()
     ignore_words = ['around', 'consulting', 'yesterday', 'for a bit', 'thanks',
                     'in-progress', 'bits of', 'even a little', 'floating',
-                    'a bit', 'reviewing']
+                    'a bit', 'reviewing', 'looking', 'advising', 'partly', 'ish',
+                    'watching', 'mostly', 'jumping']
     sep_words = ['and', ';', '+', '/', 'with special  by']
     nicknames = match.group(1)
     for word in ignore_words:
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -7,9 +7,12 @@
 
 .. branch: cppyy-packaging
 
-Upgrade to backend 1.1.0, improved handling of templated methods and
+Upgrade to backend 1.2.0, improved handling of templated methods and
 functions (in particular automatic deduction of types), improved pythonization
-interface, and a range of compatibility fixes for Python3
+interface, range of compatibility fixes for Python3, free functions now take
+fast libffi path when possible, moves for strings (incl. from Python str),
+easier/faster handling of std::vector by numpy, improved and faster object
+identity preservation
 
 .. branch: socket_default_timeout_blockingness
 
diff --git a/pypy/module/__pypy__/__init__.py b/pypy/module/__pypy__/__init__.py
--- a/pypy/module/__pypy__/__init__.py
+++ b/pypy/module/__pypy__/__init__.py
@@ -48,6 +48,7 @@
         'int_lshift':      'interp_intop.int_lshift',
         'int_rshift':      'interp_intop.int_rshift',
         'uint_rshift':     'interp_intop.uint_rshift',
+        'int_mulmod':      'interp_intop.int_mulmod',
     }
 
 
diff --git a/pypy/module/__pypy__/interp_intop.py b/pypy/module/__pypy__/interp_intop.py
--- a/pypy/module/__pypy__/interp_intop.py
+++ b/pypy/module/__pypy__/interp_intop.py
@@ -2,7 +2,7 @@
 from rpython.rtyper.lltypesystem import lltype
 from rpython.rtyper.lltypesystem.lloperation import llop
 from rpython.rlib.rarithmetic import r_uint, intmask
-from rpython.rlib.rarithmetic import int_c_div, int_c_mod
+from rpython.rlib.rarithmetic import int_c_div, int_c_mod, mulmod
 from rpython.rlib import jit
 
 
@@ -39,3 +39,7 @@
     n = r_uint(n)
     x = llop.uint_rshift(lltype.Unsigned, n, m)
     return space.newint(intmask(x))
+
+ at unwrap_spec(a=int, b=int, c=int)
+def int_mulmod(space, a, b, c):
+    return space.newint(mulmod(a, b, c))
diff --git a/pypy/module/__pypy__/test/test_intop.py b/pypy/module/__pypy__/test/test_intop.py
--- a/pypy/module/__pypy__/test/test_intop.py
+++ b/pypy/module/__pypy__/test/test_intop.py
@@ -102,3 +102,7 @@
         assert intop.uint_rshift(-1, 1) == sys.maxsize
         assert intop.uint_rshift(-1, bits-2) == 3
         assert intop.uint_rshift(-1, bits-1) == 1
+
+    def test_mulmod(self):
+        from __pypy__ import intop
+        assert intop.int_mulmod(9373891, 9832739, 2**31-1) == 1025488209
diff --git a/pypy/module/_cffi_backend/__init__.py b/pypy/module/_cffi_backend/__init__.py
--- a/pypy/module/_cffi_backend/__init__.py
+++ b/pypy/module/_cffi_backend/__init__.py
@@ -3,7 +3,7 @@
 from rpython.rlib import rdynload, clibffi
 from rpython.rtyper.lltypesystem import rffi
 
-VERSION = "1.11.5"
+VERSION = "1.12.0"
 
 FFI_DEFAULT_ABI = clibffi.FFI_DEFAULT_ABI
 try:
diff --git a/pypy/module/_cffi_backend/newtype.py b/pypy/module/_cffi_backend/newtype.py
--- a/pypy/module/_cffi_backend/newtype.py
+++ b/pypy/module/_cffi_backend/newtype.py
@@ -258,6 +258,11 @@
 SF_PACKED             = 0x08
 SF_STD_FIELD_POS      = 0x80
 
+if sys.platform == 'win32':
+    SF_DEFAULT_PACKING = 8
+else:
+    SF_DEFAULT_PACKING = 0x40000000    # a huge power of two
+
 
 if sys.platform == 'win32':
     DEFAULT_SFLAGS_PLATFORM = SF_MSVC_BITFIELDS
@@ -309,10 +314,18 @@
         w_ctype._custom_field_pos = True
 
 @unwrap_spec(w_ctype=ctypeobj.W_CType, totalsize=int, totalalignment=int,
-             sflags=int)
+             sflags=int, pack=int)
 def complete_struct_or_union(space, w_ctype, w_fields, w_ignored=None,
-                             totalsize=-1, totalalignment=-1, sflags=0):
+                             totalsize=-1, totalalignment=-1, sflags=0,
+                             pack=0):
     sflags = complete_sflags(sflags)
+    if sflags & SF_PACKED:
+        pack = 1
+    elif pack <= 0:
+        pack = SF_DEFAULT_PACKING
+    else:
+        sflags |= SF_PACKED
+
     if (not isinstance(w_ctype, ctypestruct.W_CTypeStructOrUnion)
             or w_ctype.size >= 0):
         raise oefmt(space.w_TypeError,
@@ -362,7 +375,7 @@
         # update the total alignment requirement, but skip it if the
         # field is an anonymous bitfield or if SF_PACKED
         falignorg = ftype.alignof()
-        falign = 1 if sflags & SF_PACKED else falignorg
+        falign = min(pack, falignorg)
         do_align = True
         if (sflags & SF_GCC_ARM_BITFIELDS) == 0 and fbitsize >= 0:
             if (sflags & SF_MSVC_BITFIELDS) == 0:
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -1,7 +1,7 @@
 # ____________________________________________________________
 
 import sys
-assert __version__ == "1.11.5", ("This test_c.py file is for testing a version"
+assert __version__ == "1.12.0", ("This test_c.py file is for testing a version"
                                  " of cffi that differs from the one that we"
                                  " get from 'import _cffi_backend'")
 if sys.version_info < (3,):
@@ -3541,30 +3541,50 @@
     BLong = new_primitive_type("long")
     BChar = new_primitive_type("char")
     BShort = new_primitive_type("short")
-    BStruct = new_struct_type("struct foo")
-    complete_struct_or_union(BStruct, [('a1', BLong, -1),
-                                       ('a2', BChar, -1),
-                                       ('a3', BShort, -1)],
-                             None, -1, -1, SF_PACKED)
-    d = BStruct.fields
-    assert len(d) == 3
-    assert d[0][0] == 'a1'
-    assert d[0][1].type is BLong
+    for extra_args in [(SF_PACKED,), (0, 1)]:
+        BStruct = new_struct_type("struct foo")
+        complete_struct_or_union(BStruct, [('a1', BLong, -1),
+                                           ('a2', BChar, -1),
+                                           ('a3', BShort, -1)],
+                                 None, -1, -1, *extra_args)
+        d = BStruct.fields
+        assert len(d) == 3
+        assert d[0][0] == 'a1'
+        assert d[0][1].type is BLong
+        assert d[0][1].offset == 0
+        assert d[0][1].bitshift == -1
+        assert d[0][1].bitsize == -1
+        assert d[1][0] == 'a2'
+        assert d[1][1].type is BChar
+        assert d[1][1].offset == sizeof(BLong)
+        assert d[1][1].bitshift == -1
+        assert d[1][1].bitsize == -1
+        assert d[2][0] == 'a3'
+        assert d[2][1].type is BShort
+        assert d[2][1].offset == sizeof(BLong) + sizeof(BChar)
+        assert d[2][1].bitshift == -1
+        assert d[2][1].bitsize == -1
+        assert sizeof(BStruct) == sizeof(BLong) + sizeof(BChar) + sizeof(BShort)
+        assert alignof(BStruct) == 1
+    #
+    BStruct2 = new_struct_type("struct foo")
+    complete_struct_or_union(BStruct2, [('b1', BChar, -1),
+                                        ('b2', BLong, -1)],
+                             None, -1, -1, 0, 2)
+    d = BStruct2.fields
+    assert len(d) == 2
+    assert d[0][0] == 'b1'
+    assert d[0][1].type is BChar
     assert d[0][1].offset == 0
     assert d[0][1].bitshift == -1
     assert d[0][1].bitsize == -1
-    assert d[1][0] == 'a2'
-    assert d[1][1].type is BChar
-    assert d[1][1].offset == sizeof(BLong)
+    assert d[1][0] == 'b2'
+    assert d[1][1].type is BLong
+    assert d[1][1].offset == 2
     assert d[1][1].bitshift == -1
     assert d[1][1].bitsize == -1
-    assert d[2][0] == 'a3'
-    assert d[2][1].type is BShort
-    assert d[2][1].offset == sizeof(BLong) + sizeof(BChar)
-    assert d[2][1].bitshift == -1
-    assert d[2][1].bitsize == -1
-    assert sizeof(BStruct) == sizeof(BLong) + sizeof(BChar) + sizeof(BShort)
-    assert alignof(BStruct) == 1
+    assert sizeof(BStruct2) == 2 + sizeof(BLong)
+    assert alignof(BStruct2) == 2
 
 def test_packed_with_bitfields():
     if sys.platform == "win32":
@@ -3915,8 +3935,8 @@
 
 def test_char_pointer_conversion():
     import warnings
-    assert __version__.startswith(("1.8", "1.9", "1.10", "1.11")), (
-        "consider turning the warning into an error")
+    assert __version__.startswith("1."), (
+        "the warning will be an error if we ever release cffi 2.x")
     BCharP = new_pointer_type(new_primitive_type("char"))
     BIntP = new_pointer_type(new_primitive_type("int"))
     BVoidP = new_pointer_type(new_void_type())
diff --git a/pypy/module/_cppyy/converter.py b/pypy/module/_cppyy/converter.py
--- a/pypy/module/_cppyy/converter.py
+++ b/pypy/module/_cppyy/converter.py
@@ -1,15 +1,12 @@
 import sys
 
 from pypy.interpreter.error import OperationError, oefmt
-
 from rpython.rtyper.lltypesystem import rffi, lltype
 from rpython.rlib.rarithmetic import r_singlefloat, r_longfloat
 from rpython.rlib import rfloat, rawrefcount
-
 from pypy.module._rawffi.interp_rawffi import letter2tp
 from pypy.module._rawffi.array import W_ArrayInstance
-
-from pypy.module._cppyy import helper, capi, ffitypes
+from pypy.module._cppyy import helper, capi, ffitypes, lowlevelviews
 
 # Converter objects are used to translate between RPython and C++. They are
 # defined by the type name for which they provide conversion. Uses are for
@@ -82,10 +79,9 @@
 
 
 class TypeConverter(object):
-    _immutable_fields_ = ['cffi_name', 'uses_local', 'name']
+    _immutable_fields_ = ['cffi_name', 'name']
 
     cffi_name  = None
-    uses_local = False
     name       = ""
 
     def __init__(self, space, extra):
@@ -108,10 +104,10 @@
         from pypy.module._cppyy.interp_cppyy import FastCallNotPossible
         raise FastCallNotPossible
 
-    def convert_argument(self, space, w_obj, address, call_local):
+    def convert_argument(self, space, w_obj, address):
         self._is_abstract(space)
 
-    def convert_argument_libffi(self, space, w_obj, address, call_local):
+    def convert_argument_libffi(self, space, w_obj, address, scratch):
         from pypy.module._cppyy.interp_cppyy import FastCallNotPossible
         raise FastCallNotPossible
 
@@ -125,10 +121,10 @@
     def to_memory(self, space, w_obj, w_value, offset):
         self._is_abstract(space)
 
-    def finalize_call(self, space, w_obj, call_local):
+    def finalize_call(self, space, w_obj):
         pass
 
-    def free_argument(self, space, arg, call_local):
+    def free_argument(self, space, arg):
         pass
 
 
@@ -150,7 +146,8 @@
         # read access, so no copy needed
         address_value = self._get_raw_address(space, w_obj, offset)
         address = rffi.cast(rffi.ULONG, address_value)
-        return W_ArrayInstance(space, letter2tp(space, self.typecode), self.size, address)
+        return lowlevelviews.W_LowLevelView(
+            space, letter2tp(space, self.typecode), self.size, address)
 
     def to_memory(self, space, w_obj, w_value, offset):
         # copy the full array (uses byte copy for now)
@@ -172,7 +169,7 @@
         state = space.fromcache(ffitypes.State)
         return state.c_voidp
 
-    def convert_argument(self, space, w_obj, address, call_local):
+    def convert_argument(self, space, w_obj, address):
         w_tc = space.findattr(w_obj, space.newtext('typecode'))
         if w_tc is not None and space.text_w(w_tc) != self.typecode:
             raise oefmt(space.w_TypeError,
@@ -191,7 +188,8 @@
         # read access, so no copy needed
         address_value = self._get_raw_address(space, w_obj, offset)
         address = rffi.cast(rffi.ULONGP, address_value)
-        return W_ArrayInstance(space, letter2tp(space, self.typecode), self.size, address[0])
+        return lowlevelviews.W_LowLevelView(
+            space, letter2tp(space, self.typecode), self.size, address[0])
 
     def to_memory(self, space, w_obj, w_value, offset):
         # copy only the pointer value
@@ -208,7 +206,7 @@
 class NumericTypeConverterMixin(object):
     _mixin_ = True
 
-    def convert_argument_libffi(self, space, w_obj, address, call_local):
+    def convert_argument_libffi(self, space, w_obj, address, scratch):
         x = rffi.cast(self.c_ptrtype, address)
         x[0] = self._unwrap_object(space, w_obj)
 
@@ -228,26 +226,23 @@
 
 class ConstRefNumericTypeConverterMixin(NumericTypeConverterMixin):
     _mixin_ = True
-    _immutable_fields_ = ['uses_local']
-
-    uses_local = True
 
     def cffi_type(self, space):
         state = space.fromcache(ffitypes.State)
         return state.c_voidp
 
-    def convert_argument_libffi(self, space, w_obj, address, call_local):
-        assert rffi.sizeof(self.c_type) <= 2*rffi.sizeof(rffi.VOIDP)  # see interp_cppyy.py
+    def convert_argument_libffi(self, space, w_obj, address, scratch):
         obj = self._unwrap_object(space, w_obj)
-        typed_buf = rffi.cast(self.c_ptrtype, call_local)
+        typed_buf = rffi.cast(self.c_ptrtype, scratch)
         typed_buf[0] = obj
         x = rffi.cast(rffi.VOIDPP, address)
-        x[0] = call_local
+        x[0] = scratch
+
 
 class IntTypeConverterMixin(NumericTypeConverterMixin):
     _mixin_ = True
 
-    def convert_argument(self, space, w_obj, address, call_local):
+    def convert_argument(self, space, w_obj, address):
         x = rffi.cast(self.c_ptrtype, address)
         x[0] = self._unwrap_object(space, w_obj)
         ba = rffi.cast(rffi.CCHARP, address)
@@ -256,7 +251,7 @@
 class FloatTypeConverterMixin(NumericTypeConverterMixin):
     _mixin_ = True
 
-    def convert_argument(self, space, w_obj, address, call_local):
+    def convert_argument(self, space, w_obj, address):
         x = rffi.cast(self.c_ptrtype, address)
         x[0] = self._unwrap_object(space, w_obj)
         ba = rffi.cast(rffi.CCHARP, address)
@@ -273,18 +268,18 @@
         state = space.fromcache(ffitypes.State)
         return state.c_void
 
-    def convert_argument(self, space, w_obj, address, call_local):
+    def convert_argument(self, space, w_obj, address):
         self._is_abstract(space)
 
 
 class BoolConverter(ffitypes.typeid(bool), TypeConverter):
-    def convert_argument(self, space, w_obj, address, call_local):
+    def convert_argument(self, space, w_obj, address):
         x = rffi.cast(rffi.LONGP, address)
         x[0] = self._unwrap_object(space, w_obj)
         ba = rffi.cast(rffi.CCHARP, address)
         ba[capi.c_function_arg_typeoffset(space)] = 'b'
 
-    def convert_argument_libffi(self, space, w_obj, address, call_local):
+    def convert_argument_libffi(self, space, w_obj, address, scratch):
         x = rffi.cast(rffi.LONGP, address)
         x[0] = self._unwrap_object(space, w_obj)
 
@@ -303,13 +298,13 @@
             address[0] = '\x00'
 
 class CharConverter(ffitypes.typeid(rffi.CHAR), TypeConverter):
-    def convert_argument(self, space, w_obj, address, call_local):
+    def convert_argument(self, space, w_obj, address):
         x = rffi.cast(rffi.CCHARP, address)
         x[0] = self._unwrap_object(space, w_obj)
         ba = rffi.cast(rffi.CCHARP, address)
         ba[capi.c_function_arg_typeoffset(space)] = 'b'
 
-    def convert_argument_libffi(self, space, w_obj, address, call_local):
+    def convert_argument_libffi(self, space, w_obj, address, scratch):
         x = rffi.cast(self.c_ptrtype, address)
         x[0] = self._unwrap_object(space, w_obj)
 
@@ -348,7 +343,7 @@
         state = space.fromcache(ffitypes.State)
         return state.c_voidp
 
-    def convert_argument_libffi(self, space, w_obj, address, call_local):
+    def convert_argument_libffi(self, space, w_obj, address, scratch):
         from pypy.module._cppyy.interp_cppyy import FastCallNotPossible
         raise FastCallNotPossible
 
@@ -381,7 +376,7 @@
 
 
 class CStringConverter(TypeConverter):
-    def convert_argument(self, space, w_obj, address, call_local):
+    def convert_argument(self, space, w_obj, address):
         x = rffi.cast(rffi.LONGP, address)
         arg = space.text_w(w_obj)
         x[0] = rffi.cast(rffi.LONG, rffi.str2charp(arg))
@@ -393,7 +388,7 @@
         charpptr = rffi.cast(rffi.CCHARPP, address)
         return space.newtext(rffi.charp2str(charpptr[0]))
 
-    def free_argument(self, space, arg, call_local):
+    def free_argument(self, space, arg):
         lltype.free(rffi.cast(rffi.CCHARPP, arg)[0], flavor='raw')
 
 class CStringConverterWithSize(CStringConverter):
@@ -423,13 +418,13 @@
         state = space.fromcache(ffitypes.State)
         return state.c_voidp
 
-    def convert_argument(self, space, w_obj, address, call_local):
+    def convert_argument(self, space, w_obj, address):
         x = rffi.cast(rffi.VOIDPP, address)
         x[0] = self._unwrap_object(space, w_obj)
         ba = rffi.cast(rffi.CCHARP, address)
         ba[capi.c_function_arg_typeoffset(space)] = 'o'
 
-    def convert_argument_libffi(self, space, w_obj, address, call_local):
+    def convert_argument_libffi(self, space, w_obj, address, scratch):
         x = rffi.cast(rffi.VOIDPP, address)
         x[0] = self._unwrap_object(space, w_obj)
 
@@ -442,7 +437,7 @@
             from pypy.module._cppyy import interp_cppyy
             return interp_cppyy.get_nullptr(space)
         shape = letter2tp(space, 'P')
-        return W_ArrayInstance(space, shape, sys.maxint/shape.size, ptrval)
+        return lowlevelviews.W_LowLevelView(space, shape, sys.maxint/shape.size, ptrval)
 
     def to_memory(self, space, w_obj, w_value, offset):
         address = rffi.cast(rffi.VOIDPP, self._get_raw_address(space, w_obj, offset))
@@ -452,37 +447,39 @@
             address[0] = rffi.cast(rffi.VOIDP, self._unwrap_object(space, w_value))
 
 class VoidPtrPtrConverter(TypeConverter):
-    _immutable_fields_ = ['uses_local', 'typecode']
+    typecode = 'p'
 
-    uses_local = True
-    typecode = 'a'
+    def __init__(self, space, extra):
+        self.ref_buffer = lltype.nullptr(rffi.VOIDPP.TO)
 
-    def convert_argument(self, space, w_obj, address, call_local):
+    def convert_argument(self, space, w_obj, address):
         x = rffi.cast(rffi.VOIDPP, address)
-        ba = rffi.cast(rffi.CCHARP, address)
         try:
             x[0] = get_rawbuffer(space, w_obj)
         except TypeError:
-            r = rffi.cast(rffi.VOIDPP, call_local)
-            r[0] = rffi.cast(rffi.VOIDP, get_rawobject(space, w_obj))
-            x[0] = rffi.cast(rffi.VOIDP, call_local)
+            ptr = rffi.cast(rffi.VOIDP, get_rawobject(space, w_obj))
+            self.ref_buffer = lltype.malloc(rffi.VOIDPP.TO, 1, flavor='raw')
+            self.ref_buffer[0] = ptr
+            x[0] = self.ref_buffer
+        ba = rffi.cast(rffi.CCHARP, address)
         ba[capi.c_function_arg_typeoffset(space)] = self.typecode
 
-    def finalize_call(self, space, w_obj, call_local):
-        r = rffi.cast(rffi.VOIDPP, call_local)
-        try:
-            set_rawobject(space, w_obj, r[0])
-        except OperationError:
-            pass             # no set on buffer/array/None
+    def finalize_call(self, space, w_obj):
+        if self.ref_buffer:
+            set_rawobject(space, w_obj, self.ref_buffer[0])
+
+    def free_argument(self, space, arg):
+        if self.ref_buffer:
+            lltype.free(self.ref_buffer, flavor='raw')
+            self.ref_buffer = lltype.nullptr(rffi.VOIDPP.TO)
 
 class VoidPtrRefConverter(VoidPtrPtrConverter):
-    _immutable_fields_ = ['uses_local', 'typecode']
-    uses_local = True
+    _immutable_fields_ = ['typecode']
     typecode   = 'V'
 
 class InstanceRefConverter(TypeConverter):
     _immutable_fields_ = ['typecode', 'clsdecl']
-    typecode    = 'V'
+    typecode = 'V'
 
     def __init__(self, space, clsdecl):
         from pypy.module._cppyy.interp_cppyy import W_CPPClassDecl
@@ -493,7 +490,7 @@
         from pypy.module._cppyy.interp_cppyy import W_CPPInstance
         if isinstance(w_obj, W_CPPInstance):
             from pypy.module._cppyy.interp_cppyy import INSTANCE_FLAGS_IS_RVALUE
-            if w_obj.flags & INSTANCE_FLAGS_IS_RVALUE:
+            if w_obj.rt_flags & INSTANCE_FLAGS_IS_RVALUE:
                 # reject moves as all are explicit
                 raise ValueError("lvalue expected")
             if capi.c_is_subtype(space, w_obj.clsdecl, self.clsdecl):
@@ -508,14 +505,14 @@
         state = space.fromcache(ffitypes.State)
         return state.c_voidp
 
-    def convert_argument(self, space, w_obj, address, call_local):
+    def convert_argument(self, space, w_obj, address):
         x = rffi.cast(rffi.VOIDPP, address)
         x[0] = rffi.cast(rffi.VOIDP, self._unwrap_object(space, w_obj))
         address = rffi.cast(capi.C_OBJECT, address)
         ba = rffi.cast(rffi.CCHARP, address)
         ba[capi.c_function_arg_typeoffset(space)] = self.typecode
 
-    def convert_argument_libffi(self, space, w_obj, address, call_local):
+    def convert_argument_libffi(self, space, w_obj, address, scratch):
         x = rffi.cast(rffi.VOIDPP, address)
         x[0] = rffi.cast(rffi.VOIDP, self._unwrap_object(space, w_obj))
 
@@ -525,21 +522,21 @@
         from pypy.module._cppyy.interp_cppyy import W_CPPInstance, INSTANCE_FLAGS_IS_RVALUE
         obj = space.interp_w(W_CPPInstance, w_obj)
         if obj:
-            if obj.flags & INSTANCE_FLAGS_IS_RVALUE:
-                obj.flags &= ~INSTANCE_FLAGS_IS_RVALUE
+            if obj.rt_flags & INSTANCE_FLAGS_IS_RVALUE:
+                obj.rt_flags &= ~INSTANCE_FLAGS_IS_RVALUE
                 try:
                     return InstanceRefConverter._unwrap_object(self, space, w_obj)
                 except Exception:
                     # TODO: if the method fails on some other converter, then the next
                     # overload can not be an rvalue anymore
-                    obj.flags |= INSTANCE_FLAGS_IS_RVALUE
+                    obj.rt_flags |= INSTANCE_FLAGS_IS_RVALUE
                     raise
         raise oefmt(space.w_ValueError, "object is not an rvalue")
 
 
 class InstanceConverter(InstanceRefConverter):
 
-    def convert_argument_libffi(self, space, w_obj, address, call_local):
+    def convert_argument_libffi(self, space, w_obj, address, scratch):
         from pypy.module._cppyy.interp_cppyy import FastCallNotPossible
         raise FastCallNotPossible       # TODO: by-value is a jit_libffi special case
 
@@ -551,9 +548,8 @@
     def to_memory(self, space, w_obj, w_value, offset):
         self._is_abstract(space)
 
-
 class InstancePtrConverter(InstanceRefConverter):
-    typecode    = 'o'
+    typecode = 'o'
 
     def _unwrap_object(self, space, w_obj):
         try:
@@ -569,43 +565,62 @@
         from pypy.module._cppyy import interp_cppyy
         return interp_cppyy.wrap_cppinstance(space, address, self.clsdecl, do_cast=False)
 
-    def to_memory(self, space, w_obj, w_value, offset):
-        address = rffi.cast(rffi.VOIDPP, self._get_raw_address(space, w_obj, offset))
-        address[0] = rffi.cast(rffi.VOIDP, self._unwrap_object(space, w_value))
+class InstancePtrPtrConverter(InstancePtrConverter):
+    typecode = 'o'
 
-class InstancePtrPtrConverter(InstancePtrConverter):
-    _immutable_fields_ = ['uses_local']
+    def __init__(self, space, extra):
+        InstancePtrConverter.__init__(self, space, extra)
+        self.ref_buffer = lltype.nullptr(rffi.VOIDPP.TO)
 
-    uses_local = True
+    def convert_argument(self, space, w_obj, address):
+        x = rffi.cast(rffi.VOIDPP, address)
+        ptr = rffi.cast(rffi.VOIDP, self._unwrap_object(space, w_obj))
+        self.ref_buffer = lltype.malloc(rffi.VOIDPP.TO, 1, flavor='raw')
+        self.ref_buffer[0] = ptr
+        x[0] = self.ref_buffer
+        ba = rffi.cast(rffi.CCHARP, address)
+        ba[capi.c_function_arg_typeoffset(space)] = self.typecode
 
-    def convert_argument(self, space, w_obj, address, call_local):
-        r = rffi.cast(rffi.VOIDPP, call_local)
-        r[0] = rffi.cast(rffi.VOIDP, self._unwrap_object(space, w_obj))
-        x = rffi.cast(rffi.VOIDPP, address)
-        x[0] = rffi.cast(rffi.VOIDP, call_local)
-        address = rffi.cast(capi.C_OBJECT, address)
-        ba = rffi.cast(rffi.CCHARP, address)
-        ba[capi.c_function_arg_typeoffset(space)] = 'o'
-
-    def convert_argument_libffi(self, space, w_obj, address, call_local):
+    def convert_argument_libffi(self, space, w_obj, address, scratch):
         # TODO: finalize_call not yet called for fast call (see interp_cppyy.py)
         from pypy.module._cppyy.interp_cppyy import FastCallNotPossible
         raise FastCallNotPossible
 
-    def finalize_call(self, space, w_obj, call_local):
-        from pypy.module._cppyy.interp_cppyy import W_CPPInstance
-        assert isinstance(w_obj, W_CPPInstance)
-        r = rffi.cast(rffi.VOIDPP, call_local)
-        w_obj._rawobject = rffi.cast(capi.C_OBJECT, r[0])
-
     def from_memory(self, space, w_obj, w_pycppclass, offset):
         address = rffi.cast(capi.C_OBJECT, self._get_raw_address(space, w_obj, offset))
         from pypy.module._cppyy import interp_cppyy
         return interp_cppyy.wrap_cppinstance(
             space, address, self.clsdecl, do_cast=False, is_ref=True)
 
+    def to_memory(self, space, w_obj, w_value, offset):
+        # the actual data member is of object* type, but we receive a pointer to that
+        # data member in order to modify its value, so by convention, the internal type
+        # used is object**
+        address = rffi.cast(rffi.VOIDPP, self._get_raw_address(space, w_obj, offset))
+        from pypy.module._cppyy.interp_cppyy import W_CPPInstance
+        cppinstance = space.interp_w(W_CPPInstance, w_value, can_be_None=True)
+        if cppinstance:
+            rawobject = cppinstance.get_rawobject()
+            offset = capi.c_base_offset(space, cppinstance.clsdecl, self.clsdecl, rawobject, 1)
+            obj_address = capi.direct_ptradd(rawobject, offset)
+            address[0] = rffi.cast(rffi.VOIDP, obj_address);
+            # register the value for potential recycling
+            from pypy.module._cppyy.interp_cppyy import memory_regulator
+            memory_regulator.register(cppinstance)
+        else:
+            raise oefmt(space.w_TypeError,
+                "cannot pass %T instance as %s", w_value, self.clsdecl.name)
+
+    def finalize_call(self, space, w_obj):
+        if self.ref_buffer:
+            set_rawobject(space, w_obj, self.ref_buffer[0])
+
+    def free_argument(self, space, arg):
+        if self.ref_buffer:
+            lltype.free(self.ref_buffer, flavor='raw')
+            self.ref_buffer = lltype.nullptr(rffi.VOIDPP.TO)
+
 class StdStringConverter(InstanceConverter):
-
     def __init__(self, space, extra):
         from pypy.module._cppyy import interp_cppyy
         cppclass = interp_cppyy.scope_byname(space, capi.std_string_name)
@@ -628,9 +643,37 @@
         except Exception:
             InstanceConverter.to_memory(self, space, w_obj, w_value, offset)
 
-    def free_argument(self, space, arg, call_local):
+    def free_argument(self, space, arg):
         capi.c_destruct(space, self.clsdecl, rffi.cast(capi.C_OBJECT, rffi.cast(rffi.VOIDPP, arg)[0]))
 
+class StdStringMoveConverter(StdStringConverter):
+    def _unwrap_object(self, space, w_obj):
+        # moving is same as by-ref, but have to check that move is allowed
+        moveit_reason = 3
+        from pypy.module._cppyy.interp_cppyy import W_CPPInstance, INSTANCE_FLAGS_IS_RVALUE
+        try:
+            obj = space.interp_w(W_CPPInstance, w_obj)
+            if obj and obj.rt_flags & INSTANCE_FLAGS_IS_RVALUE:
+                obj.rt_flags &= ~INSTANCE_FLAGS_IS_RVALUE
+                moveit_reason = 1
+            else:
+                moveit_reason = 0
+        except:
+            pass
+
+        if moveit_reason:
+            try:
+                return StdStringConverter._unwrap_object(self, space, w_obj)
+            except Exception:
+                 if moveit_reason == 1:
+                    # TODO: if the method fails on some other converter, then the next
+                    # overload can not be an rvalue anymore
+                    obj = space.interp_w(W_CPPInstance, w_obj)
+                    obj.rt_flags |= INSTANCE_FLAGS_IS_RVALUE
+                    raise
+
+        raise oefmt(space.w_ValueError, "object is not an rvalue")
+
 class StdStringRefConverter(InstancePtrConverter):
     _immutable_fields_ = ['cppclass', 'typecode']
     typecode    = 'V'
@@ -646,7 +689,7 @@
         state = space.fromcache(ffitypes.State)
         return state.c_voidp
 
-    def convert_argument(self, space, w_obj, address, call_local):
+    def convert_argument(self, space, w_obj, address):
         if hasattr(space, "fake"):
             raise NotImplementedError
         space.getbuiltinmodule("cpyext")
@@ -657,7 +700,7 @@
         ba = rffi.cast(rffi.CCHARP, address)
         ba[capi.c_function_arg_typeoffset(space)] = 'a'
 
-    def convert_argument_libffi(self, space, w_obj, address, call_local):
+    def convert_argument_libffi(self, space, w_obj, address, scratch):
         # TODO: free_argument not yet called for fast call (see interp_cppyy.py)
         from pypy.module._cppyy.interp_cppyy import FastCallNotPossible
         raise FastCallNotPossible
@@ -671,7 +714,7 @@
         x = rffi.cast(rffi.VOIDPP, address)
         x[0] = rffi.cast(rffi.VOIDP, ref)"""
 
-    def free_argument(self, space, arg, call_local):
+    def free_argument(self, space, arg):
         if hasattr(space, "fake"):
             raise NotImplementedError
         space.getbuiltinmodule("cpyext")
@@ -685,7 +728,7 @@
     def __init__(self, space, signature):
         self.signature = signature
 
-    def convert_argument(self, space, w_obj, address, call_local):
+    def convert_argument(self, space, w_obj, address):
         # TODO: atm, does not actually get an overload, but a staticmethod
         from pypy.module._cppyy.interp_cppyy import W_CPPOverload
         cppol = space.interp_w(W_CPPOverload, w_obj)
@@ -740,7 +783,7 @@
         raise oefmt(space.w_TypeError,
                     "cannot pass %T instance as %s", w_obj, self.rawdecl.name)
 
-    def convert_argument(self, space, w_obj, address, call_local):
+    def convert_argument(self, space, w_obj, address):
         x = rffi.cast(rffi.VOIDPP, address)
         x[0] = rffi.cast(rffi.VOIDP, self._unwrap_object(space, w_obj))
         address = rffi.cast(capi.C_OBJECT, address)
@@ -883,6 +926,7 @@
 _converters["std::basic_string<char>"]           = StdStringConverter
 _converters["const std::basic_string<char>&"]    = StdStringConverter     # TODO: shouldn't copy
 _converters["std::basic_string<char>&"]          = StdStringRefConverter
+_converters["std::basic_string<char>&&"]         = StdStringMoveConverter
 
 _converters["PyObject*"]                         = PyObjectConverter
 
@@ -1000,6 +1044,7 @@
         ("std::basic_string<char>",         "string"),
         ("const std::basic_string<char>&",  "const string&"),
         ("std::basic_string<char>&",        "string&"),
+        ("std::basic_string<char>&&",       "string&&"),
 
         ("PyObject*",                       "_object*"),
     )
diff --git a/pypy/module/_cppyy/executor.py b/pypy/module/_cppyy/executor.py
--- a/pypy/module/_cppyy/executor.py
+++ b/pypy/module/_cppyy/executor.py
@@ -1,14 +1,10 @@
 import sys
 
 from pypy.interpreter.error import oefmt
-
 from rpython.rtyper.lltypesystem import rffi, lltype
 from rpython.rlib import jit_libffi
-
 from pypy.module._rawffi.interp_rawffi import letter2tp
-from pypy.module._rawffi.array import W_Array, W_ArrayInstance
-
-from pypy.module._cppyy import helper, capi, ffitypes
+from pypy.module._cppyy import helper, capi, ffitypes, lowlevelviews
 
 # Executor objects are used to dispatch C++ methods. They are defined by their
 # return type only: arguments are converted by Converter objects, and Executors
@@ -60,7 +56,7 @@
             from pypy.module._cppyy import interp_cppyy
             return interp_cppyy.get_nullptr(space)
         shape = letter2tp(space, self.typecode)
-        return W_ArrayInstance(space, shape, sys.maxint/shape.size, ptrval)
+        return lowlevelviews.W_LowLevelView(space, shape, sys.maxint/shape.size, ptrval)
 
 
 class VoidExecutor(Executor):
@@ -98,10 +94,10 @@
     def __init__(self, space, extra):
         Executor.__init__(self, space, extra)
         self.do_assign = False
-        self.item = rffi.cast(self.c_type, 0)
+        self.w_item = space.w_None
 
     def set_item(self, space, w_item):
-        self.item = self._unwrap_object(space, w_item)
+        self.w_item = w_item
         self.do_assign = True
 
     #def _wrap_object(self, space, obj):
@@ -109,7 +105,7 @@
 
     def _wrap_reference(self, space, rffiptr):
         if self.do_assign:
-            rffiptr[0] = self.item
+            rffiptr[0] = rffi.cast(self.c_type, self._unwrap_object(space, self.w_item))
         self.do_assign = False
         return self._wrap_object(space, rffiptr[0])    # all paths, for rtyper
 
diff --git a/pypy/module/_cppyy/interp_cppyy.py b/pypy/module/_cppyy/interp_cppyy.py
--- a/pypy/module/_cppyy/interp_cppyy.py
+++ b/pypy/module/_cppyy/interp_cppyy.py
@@ -1,6 +1,9 @@
 import pypy.module._cppyy.capi as capi
 
 from pypy.interpreter.error import OperationError, oefmt
+from pypy.interpreter.function import Method
+from pypy.interpreter.argument import Arguments
+from pypy.interpreter.typedef import interp_attrproperty_w, descr_generic_ne, make_weakref_descr
 from pypy.interpreter.gateway import interp2app, unwrap_spec
 from pypy.interpreter.typedef import TypeDef, GetSetProperty, interp_attrproperty
 from pypy.interpreter.baseobjspace import W_Root
@@ -38,6 +41,7 @@
              'void**'        : 100,
              'float'         :  30,
              'double'        :  10,
+             'bool'          :   1,
              'const string&' :   1, } # solves a specific string ctor overload
 
 from rpython.rlib.listsort import make_timsort_class
@@ -164,13 +168,16 @@
 #
 #  W_CPPOverload:                 instance methods (base class)
 #  W_CPPConstructorOverload:      constructors
+#  W_CPPAbstractCtorOverload:     to provent instantiation of abstract classes
 #  W_CPPStaticOverload:           free and static functions
 #  W_CPPTemplateOverload:         templated methods
-#  W_CPPTemplateStaticOveload:    templated free and static functions
+#  W_CPPTemplateStaticOverload:   templated free and static functions
 #
 #  CPPMethod:         a single function or method (base class)
 #  CPPSetItem:        specialization for Python's __setitem__
 #
+#  MethodWithProps:   python instancemethod that forwards properties
+#
 # All methods/functions derive from CPPMethod and are collected as overload
 # candidates in user-facing overload classes. Templated methods are a two-step
 # process, where first the template is instantiated (or selected if already
@@ -183,13 +190,13 @@
     the memory_regulator."""
 
     _attrs_ = ['space', 'scope', 'cppmethod', 'arg_defs', 'args_required',
-               'converters', 'executor', '_funcaddr', 'cif_descr', 'uses_local']
+               'converters', 'executor', '_funcaddr', 'cif_descr']
     _immutable_fields_ = ['scope', 'cppmethod', 'arg_defs', 'args_required',
-                          'converters', 'executor', 'uses_local']
+               'converters', 'executor', '_funcaddr', 'cif_descr']
 
-    def __init__(self, space, declaring_scope, cppmethod, arg_defs, args_required):
+    def __init__(self, space, decl_scope, cppmethod, arg_defs, args_required):
         self.space = space
-        self.scope = declaring_scope
+        self.scope = decl_scope
         self.cppmethod = cppmethod
         self.arg_defs = arg_defs
         self.args_required = args_required
@@ -200,14 +207,6 @@
         self.executor = None
         self.cif_descr = lltype.nullptr(jit_libffi.CIF_DESCRIPTION)
         self._funcaddr = lltype.nullptr(capi.C_FUNC_PTR.TO)
-        self.uses_local = False
-
-    def _address_from_local_buffer(self, call_local, idx):
-        if not call_local:
-            return call_local
-        stride = 2*rffi.sizeof(rffi.VOIDP)
-        loc_idx = lltype.direct_ptradd(rffi.cast(rffi.CCHARP, call_local), idx*stride)
-        return rffi.cast(rffi.VOIDP, loc_idx)
 
     @jit.unroll_safe
     def call(self, cppthis, args_w, useffi):
@@ -233,61 +232,56 @@
             except Exception:
                 pass
 
-        # some calls, e.g. for ptr-ptr or reference need a local array to store data for
-        # the duration of the call
-        if self.uses_local:
-            call_local = lltype.malloc(rffi.VOIDP.TO, 2*len(args_w), flavor='raw')
-        else:
-            call_local = lltype.nullptr(rffi.VOIDP.TO)
+        # attempt to call directly through ffi chain
+        if useffi and self._funcaddr:
+            try:
+                return self.do_fast_call(cppthis, args_w)
+            except FastCallNotPossible:
+                pass      # can happen if converters or executor does not implement ffi
 
+        # ffi chain must have failed; using stub functions instead
+        args, stat = self.prepare_arguments(args_w)
         try:
-            # attempt to call directly through ffi chain
-            if useffi and self._funcaddr:
-                try:
-                    return self.do_fast_call(cppthis, args_w, call_local)
-                except FastCallNotPossible:
-                    pass      # can happen if converters or executor does not implement ffi
-
-            # ffi chain must have failed; using stub functions instead
-            args, stat = self.prepare_arguments(args_w, call_local)
-            try:
-                result = self.executor.execute(
-                    self.space, self.cppmethod, cppthis, len(args_w), args)
-                if stat[0] != rffi.cast(rffi.ULONG, 0):
-                    what = rffi.cast(rffi.CCHARP, stat[1])
-                    pywhat = rffi.charp2str(what)
-                    capi.c_free(self.space, rffi.cast(rffi.VOIDP, what))
-                    raise OperationError(self.space.w_Exception, self.space.newtext(pywhat))
-                return result
-            finally:
-                self.finalize_call(args, args_w, call_local)
+            result = self.executor.execute(
+                self.space, self.cppmethod, cppthis, len(args_w), args)
+            if stat[0] != rffi.cast(rffi.ULONG, 0):
+                what = rffi.cast(rffi.CCHARP, stat[1])
+                pywhat = rffi.charp2str(what)
+                capi.c_free(self.space, rffi.cast(rffi.VOIDP, what))
+                raise OperationError(self.space.w_Exception, self.space.newtext(pywhat))
+            return result
         finally:
-            if call_local:
-                lltype.free(call_local, flavor='raw')
+            self.finalize_call(args, args_w)
 
     @jit.unroll_safe
-    def do_fast_call(self, cppthis, args_w, call_local):
+    def do_fast_call(self, cppthis, args_w):
         if self.cif_descr == lltype.nullptr(jit_libffi.CIF_DESCRIPTION):
             raise FastCallNotPossible
         jit.promote(self)
         cif_descr = self.cif_descr
-        buffer = lltype.malloc(rffi.CCHARP.TO, cif_descr.exchange_size, flavor='raw')
+        # add extra space for const-ref support (see converter.py)
+        buffer = lltype.malloc(rffi.CCHARP.TO,
+            cif_descr.exchange_size+len(self.arg_defs)*rffi.sizeof(rffi.DOUBLE), flavor='raw')
+        thisoff = 0
         try:
-            # this pointer
-            data = rffi.ptradd(buffer, cif_descr.exchange_args[0])
-            x = rffi.cast(rffi.LONGP, data)       # LONGP needed for test_zjit.py
-            x[0] = rffi.cast(rffi.LONG, cppthis)
+            if cppthis:
+                # this pointer
+                data = rffi.ptradd(buffer, cif_descr.exchange_args[0])
+                x = rffi.cast(rffi.LONGP, data)       # LONGP needed for test_zjit.py
+                x[0] = rffi.cast(rffi.LONG, cppthis)
+                thisoff = 1
 
-            # other arguments and defaults
-            i = len(self.arg_defs) + 1
+            # actual provided arguments
+            i = -1      # needed if all arguments are defaults
             for i in range(len(args_w)):
                 conv = self.converters[i]
-                w_arg = args_w[i]
-                data = rffi.ptradd(buffer, cif_descr.exchange_args[i+1])
-                conv.convert_argument_libffi(self.space, w_arg, data, call_local)
+                data = rffi.ptradd(buffer, cif_descr.exchange_args[i+thisoff])
+                scratch = rffi.ptradd(buffer, cif_descr.exchange_size+i*rffi.sizeof(rffi.DOUBLE))
+                conv.convert_argument_libffi(self.space, args_w[i], data, scratch)
+            # drop in defaults for the rest
             for j in range(i+1, len(self.arg_defs)):
                 conv = self.converters[j]
-                data = rffi.ptradd(buffer, cif_descr.exchange_args[j+1])
+                data = rffi.ptradd(buffer, cif_descr.exchange_args[j+thisoff])
                 conv.default_argument_libffi(self.space, data)
 
             assert self._funcaddr
@@ -346,22 +340,17 @@
         self.executor = executor.get_executor(
             self.space, capi.c_method_result_type(self.space, self.cppmethod))
 
-        for conv in self.converters:
-            if conv.uses_local:
-                self.uses_local = True
-                break
-
         # Each CPPMethod corresponds one-to-one to a C++ equivalent and cppthis
         # has been offset to the matching class. Hence, the libffi pointer is
         # uniquely defined and needs to be setup only once.
         funcaddr = capi.c_function_address(self.space, self.cppmethod)
-        if funcaddr and cppthis:      # TODO: methods only for now
+        if funcaddr:
             state = self.space.fromcache(ffitypes.State)
 
-            # argument type specification (incl. cppthis)
+            # argument type specification (incl. cppthis if applicable)
             fargs = []
             try:
-                fargs.append(state.c_voidp)
+                if cppthis: fargs.append(state.c_voidp)
                 for i, conv in enumerate(self.converters):
                     fargs.append(conv.cffi_type(self.space))
                 fresult = self.executor.cffi_type(self.space)
@@ -386,7 +375,7 @@
             self._funcaddr = funcaddr
 
     @jit.unroll_safe
-    def prepare_arguments(self, args_w, call_local):
+    def prepare_arguments(self, args_w):
         args = capi.c_allocate_function_args(self.space, len(args_w))
         stride = capi.c_function_arg_sizeof(self.space)
         for i in range(len(args_w)):
@@ -394,15 +383,13 @@
             w_arg = args_w[i]
             try:
                 arg_i = lltype.direct_ptradd(rffi.cast(rffi.CCHARP, args), i*stride)
-                loc_i = self._address_from_local_buffer(call_local, i)
-                conv.convert_argument(self.space, w_arg, rffi.cast(capi.C_OBJECT, arg_i), loc_i)
+                conv.convert_argument(self.space, w_arg, rffi.cast(capi.C_OBJECT, arg_i))
             except:
                 # fun :-(
                 for j in range(i):
                     conv = self.converters[j]
                     arg_j = lltype.direct_ptradd(rffi.cast(rffi.CCHARP, args), j*stride)
-                    loc_j = self._address_from_local_buffer(call_local, j)
-                    conv.free_argument(self.space, rffi.cast(capi.C_OBJECT, arg_j), loc_j)
+                    conv.free_argument(self.space, rffi.cast(capi.C_OBJECT, arg_j))
                 capi.c_deallocate_function_args(self.space, args)
                 raise
         stat = rffi.cast(rffi.ULONGP,
@@ -411,14 +398,13 @@
         return args, stat
 
     @jit.unroll_safe
-    def finalize_call(self, args, args_w, call_local):
+    def finalize_call(self, args, args_w):
         stride = capi.c_function_arg_sizeof(self.space)
         for i in range(len(args_w)):
             conv = self.converters[i]
             arg_i = lltype.direct_ptradd(rffi.cast(rffi.CCHARP, args), i*stride)
-            loc_i = self._address_from_local_buffer(call_local, i)
-            conv.finalize_call(self.space, args_w[i], loc_i)
-            conv.free_argument(self.space, rffi.cast(capi.C_OBJECT, arg_i), loc_i)
+            conv.finalize_call(self.space, args_w[i])
+            conv.free_argument(self.space, rffi.cast(capi.C_OBJECT, arg_i))
         capi.c_deallocate_function_args(self.space, args)
 
     def signature(self, show_formalargs=True):
@@ -429,8 +415,10 @@
 
     def priority(self):
         total_arg_priority = 0
-        for p in [priority.get(arg_type, 0) for arg_type, arg_dflt in self.arg_defs]:
-            total_arg_priority += p
+        for arg_type, arg_dflt in self.arg_defs:
+            total_arg_priority += priority.get(arg_type, 0)
+            if '&&' in arg_type:
+                total_arg_priority += 100
         return total_arg_priority
 
     @rgc.must_be_light_finalizer
@@ -450,7 +438,7 @@
 
 class CPPSetItem(CPPMethod):
     """Method dispatcher specific to Python's __setitem__ mapped onto C++'s
-    operator[](int). The former function takes an extra argument to assign to
+    operator[](T). The former function takes an extra argument to assign to
     the return type of the latter."""
 
     _attrs_ = []
@@ -466,42 +454,81 @@
         CPPMethod.call(self, cppthis, args_w, useffi)
 
 
+# CPPOverloads have settable flags that control memory and ffi behavior. These flags
+# need forwarding, which the normal instancemethod does not provide, hence this
+# derived class.
+class MethodWithProps(Method):
+    # allow user to determine ffi use rules per overload
+    def fget_useffi(self, space):
+        f = space.interp_w(W_CPPOverload, self.w_function)
+        return f.fget_useffi(space)
+
+    @unwrap_spec(value=bool)
+    def fset_useffi(self, space, value):
+        f = space.interp_w(W_CPPOverload, self.w_function)
+        f.fset_useffi(space, value)
+
+MethodWithProps.typedef = TypeDef(
+    "cpp_instancemethod",
+    __doc__ = """cpp_instancemethod(function, instance, class)
+
+Create an instance method object.""",
+    __new__ = interp2app(MethodWithProps.descr_method__new__.im_func),
+    __call__ = interp2app(MethodWithProps.descr_method_call),
+    __get__ = interp2app(MethodWithProps.descr_method_get),
+    __func__ = interp_attrproperty_w('w_function', cls=MethodWithProps),
+    __self__ = interp_attrproperty_w('w_instance', cls=MethodWithProps),
+    __getattribute__ = interp2app(MethodWithProps.descr_method_getattribute),
+    __eq__ = interp2app(MethodWithProps.descr_method_eq),
+    __ne__ = descr_generic_ne,
+    __hash__ = interp2app(MethodWithProps.descr_method_hash),
+    __repr__ = interp2app(MethodWithProps.descr_method_repr),
+    __reduce__ = interp2app(MethodWithProps.descr_method__reduce__),
+    __weakref__ = make_weakref_descr(MethodWithProps),
+    __useffi__ = GetSetProperty(MethodWithProps.fget_useffi, MethodWithProps.fset_useffi),
+    )
+MethodWithProps.typedef.acceptable_as_base_class = False
+
+
 class W_CPPOverload(W_Root):
     """App-level dispatcher: controls a collection of (potentially) overloaded methods
     or functions. Calls these in order and deals with error handling and reporting."""
 
-    _attrs_ = ['space', 'scope', 'functions', 'flags', 'w_this']
+    _attrs_ = ['space', 'scope', 'functions', 'flags']
     _immutable_fields_ = ['scope', 'functions[*]']
 
-    def __init__(self, space, declaring_scope, functions, flags = OVERLOAD_FLAGS_USE_FFI):
+    def __init__(self, space, decl_scope, funcs, flags = OVERLOAD_FLAGS_USE_FFI):
         self.space  = space
-        self.scope  = declaring_scope
+        self.scope  = decl_scope
         from rpython.rlib import debug
-        self.functions = debug.make_sure_not_resized(functions)
+        self.functions = debug.make_sure_not_resized(funcs)
         self.flags  = flags
-        self.w_this = self.space.w_None
+
+    def descr_get(self, w_obj, w_cls=None):
+        """functionobject.__get__(obj[, type]) -> method"""
+        # TODO: check validity of w_cls if given
+        # TODO: this does not work for Python 3, which does not have
+        #  unbound methods (probably no common code possible, see also
+        #  pypy/interpreter/function.py)
+        space = self.space
+        asking_for_bound = (space.is_none(w_cls) or
+                            not space.is_w(w_obj, space.w_None) or
+                            space.is_w(w_cls, space.type(space.w_None)))
+        if asking_for_bound:
+            return MethodWithProps(space, self, w_obj)
+        else:
+            return self   # unbound methods don't exist in Python 3
 
     @unwrap_spec(args_w='args_w')
-    def descr_get(self, w_cppinstance, args_w):
-        if self.space.is_w(w_cppinstance, self.space.w_None):
-            return self  # unbound, so no new instance needed
-        cppol = W_CPPOverload(self.space, self.scope, self.functions, self.flags)
-        cppol.w_this = w_cppinstance
-        return cppol     # bound
-
-    @unwrap_spec(args_w='args_w')
-    def call(self, args_w):
-        if self.space.is_w(self.w_this, self.space.w_None) and len(args_w):
-            w_this = args_w[0]
-            args_w = args_w[1:]
-        else:
-            w_this = self.w_this
+    def call_args(self, args_w):
+        jit.promote(self)
+        w_this = args_w[0]
         cppinstance = self.space.interp_w(W_CPPInstance, w_this)
         cppinstance._nullcheck()
         if not capi.c_is_subtype(self.space, cppinstance.clsdecl, self.scope):
             raise oefmt(self.space.w_TypeError,
                 "cannot pass %T instance as %s", w_this, self.scope.name)
-        return self.call_impl(cppinstance.get_cppthis(self.scope), args_w)
+        return self.call_impl(cppinstance.get_cppthis(self.scope), args_w[1:])
 
     @jit.unroll_safe
     def call_impl(self, cppthis, args_w):
@@ -562,6 +589,16 @@
             sig += '\n'+self.functions[i].prototype()
         return self.space.newtext(sig)
 
+    @unwrap_spec(signature='text')
+    def mp_overload(self, signature):
+        sig = '(%s)' % signature
+        for f in self.functions:
+            if f.signature(False) == sig:
+                if isinstance(self, W_CPPStaticOverload):
+                    return W_CPPStaticOverload(self.space, self.scope, [f])
+                return W_CPPOverload(self.space, self.scope, [f])
+        raise oefmt(self.space.w_LookupError, "signature '%s' not found", signature)
+
     # allow user to determine ffi use rules per overload
     def fget_useffi(self, space):
         return space.newbool(bool(self.flags & OVERLOAD_FLAGS_USE_FFI))
@@ -576,15 +613,20 @@
     def fget_doc(self, space):
         return self.prototype()
 
+    def getname(self, space): 
+        # for the benefit of Method/instancemethod
+        return capi.c_method_name(space, self.functions[0].cppmethod).decode('latin-1')
+
     def __repr__(self):
         return "W_CPPOverload(%s)" % [f.prototype() for f in self.functions]
 
 W_CPPOverload.typedef = TypeDef(
     'CPPOverload',
-    __get__    = interp2app(W_CPPOverload.descr_get),
-    __call__   = interp2app(W_CPPOverload.call),
-    __useffi__ = GetSetProperty(W_CPPOverload.fget_useffi, W_CPPOverload.fset_useffi),
-    __doc__    = GetSetProperty(W_CPPOverload.fget_doc)
+    __get__      = interp2app(W_CPPOverload.descr_get),
+    __call__     = interp2app(W_CPPOverload.call_args),
+    __useffi__   = GetSetProperty(W_CPPOverload.fget_useffi, W_CPPOverload.fset_useffi),
+    __overload__ = interp2app(W_CPPOverload.mp_overload),
+    __doc__      = GetSetProperty(W_CPPOverload.fget_doc)
 )
 
 
@@ -593,26 +635,19 @@
 class W_CPPStaticOverload(W_CPPOverload):
     _attrs_ = []
 
-    @unwrap_spec(args_w='args_w')
-    def descr_get(self, w_cppinstance, args_w):
-        if isinstance(w_cppinstance, W_CPPInstance):
+    def descr_get(self, w_obj, w_cls=None):
+        if isinstance(w_obj, W_CPPInstance):
             # two possibilities: this is a static function called on an
             # instance and w_this must not be set, or a free function rebound
             # onto a class and w_this should be set
-            cppinstance = self.space.interp_w(W_CPPInstance, w_cppinstance)
+            cppinstance = self.space.interp_w(W_CPPInstance, w_obj)
             if cppinstance.clsdecl.handle != self.scope.handle:
-                cppol = W_CPPStaticOverload(self.space, self.scope, self.functions, self.flags)
-                cppol.w_this = w_cppinstance
-                return cppol       # bound
+                return MethodWithProps(self.space, self, w_obj)    # bound
         return self      # unbound
 
     @unwrap_spec(args_w='args_w')
-    def call(self, args_w):
-        if not self.space.is_w(self.w_this, self.space.w_None):
-            # free function used as bound method, put self back into args_w
-            cppinstance = self.space.interp_w(W_CPPInstance, self.w_this)
-            cppinstance._nullcheck()
-            args_w = [self.w_this] + args_w
+    def call_args(self, args_w):
+        jit.promote(self)
         return self.call_impl(capi.C_NULL_OBJECT, args_w)
 
     def __repr__(self):
@@ -620,37 +655,26 @@
 
 W_CPPStaticOverload.typedef = TypeDef(
     'CPPStaticOverload',
-    __get__    = interp2app(W_CPPStaticOverload.descr_get),
-    __call__   = interp2app(W_CPPStaticOverload.call),
-    __useffi__ = GetSetProperty(W_CPPStaticOverload.fget_useffi, W_CPPStaticOverload.fset_useffi),
-    __doc__    = GetSetProperty(W_CPPStaticOverload.fget_doc)
+    __get__      = interp2app(W_CPPStaticOverload.descr_get),
+    __call__     = interp2app(W_CPPStaticOverload.call_args),
+    __useffi__   = GetSetProperty(W_CPPStaticOverload.fget_useffi, W_CPPStaticOverload.fset_useffi),
+    __overload__ = interp2app(W_CPPStaticOverload.mp_overload),
+    __doc__      = GetSetProperty(W_CPPStaticOverload.fget_doc)
 )
 
 
 class W_CPPConstructorOverload(W_CPPOverload):
     _attrs_ = []
 
-    @unwrap_spec(args_w='args_w')
-    def descr_get(self, w_cppinstance, args_w):
-        if self.space.is_w(w_cppinstance, self.space.w_None):
-            return self  # unbound (TODO: probably useless)
-        cppol = W_CPPConstructorOverload(self.space, self.scope, self.functions, self.flags)
-        cppol.w_this = w_cppinstance
-        return cppol     # bound
+    def __init__(self, space, decl_scope, funcs, flags = OVERLOAD_FLAGS_USE_FFI):
+        W_CPPOverload.__init__(self, space, decl_scope, funcs, flags)
+        self.flags &= ~OVERLOAD_FLAGS_USE_FFI
 
     @unwrap_spec(args_w='args_w')
-    def call(self, args_w):
-        # TODO: factor out the following:
-        if capi.c_is_abstract(self.space, self.scope.handle):
-            raise oefmt(self.space.w_TypeError,
-                        "cannot instantiate abstract class '%s'",
-                        self.scope.name)
-        if self.space.is_w(self.w_this, self.space.w_None) and len(args_w):
-            cppinstance = self.space.interp_w(W_CPPInstance, args_w[0])
-            args_w = args_w[1:]
-        else:
-            cppinstance = self.space.interp_w(W_CPPInstance, self.w_this)
-        w_result = self.call_impl(rffi.cast(capi.C_OBJECT, self.scope.handle), args_w)
+    def call_args(self, args_w):
+        jit.promote(self)
+        cppinstance = self.space.interp_w(W_CPPInstance, args_w[0])
+        w_result = self.call_impl(rffi.cast(capi.C_OBJECT, self.scope.handle), args_w[1:])
         newthis = rffi.cast(capi.C_OBJECT, self.space.uint_w(w_result))
         if cppinstance is not None:
             cppinstance._rawobject = newthis
@@ -661,9 +685,27 @@
 
 W_CPPConstructorOverload.typedef = TypeDef(
     'CPPConstructorOverload',
-    __get__    = interp2app(W_CPPConstructorOverload.descr_get),
-    __call__   = interp2app(W_CPPConstructorOverload.call),
-    __doc__    = GetSetProperty(W_CPPConstructorOverload.fget_doc)
+    __get__      = interp2app(W_CPPConstructorOverload.descr_get),
+    __call__     = interp2app(W_CPPConstructorOverload.call_args),
+    __overload__ = interp2app(W_CPPConstructorOverload.mp_overload),
+    __doc__      = GetSetProperty(W_CPPConstructorOverload.fget_doc)
+)
+
+class W_CPPAbstractCtorOverload(W_CPPOverload):
+    _attrs_ = []
+
+    @unwrap_spec(args_w='args_w')
+    def call_args(self, args_w):
+        raise oefmt(self.space.w_TypeError,
+            "cannot instantiate abstract class '%s'", self.scope.name)
+
+    def __repr__(self):
+        return "W_CPPAbstractCtorOverload"
+
+W_CPPAbstractCtorOverload.typedef = TypeDef(
+    'CPPAbstractCtorOverload',
+    __get__    = interp2app(W_CPPAbstractCtorOverload.descr_get),
+    __call__   = interp2app(W_CPPAbstractCtorOverload.call_args),
 )
 
 
@@ -686,7 +728,7 @@
                     if args_w:
                         # try to specialize the type match for the given object
                         cppinstance = self.space.interp_w(W_CPPInstance, args_w[i])
-                        if cppinstance.flags & INSTANCE_FLAGS_IS_RVALUE:
+                        if cppinstance.rt_flags & INSTANCE_FLAGS_IS_RVALUE:
                             sugar = "&&"
                         elif cppinstance.flags & INSTANCE_FLAGS_IS_REF:
                             sugar = "*"
@@ -725,7 +767,9 @@
         # try to match with run-time instantiations
         for cppol in self.master.overloads.values():
             try:
-                return cppol.descr_get(self.w_this, []).call(args_w)
+                if not self.space.is_w(self.w_this, self.space.w_None):
+                    return self.space.call_obj_args(cppol, self.w_this, Arguments(self.space, args_w))
+                return self.space.call_args(cppol, Arguments(self.space, args_w))
             except Exception:
                 pass    # completely ignore for now; have to see whether errors become confusing
 
@@ -748,7 +792,9 @@
             except KeyError:
                 self.master.overloads[fullname] = method
 
-        return method.descr_get(self.w_this, []).call(args_w)
+        if not self.space.is_w(self.w_this, self.space.w_None):
+            return self.space.call_obj_args(method, self.w_this, Arguments(self.space, args_w))
+        return self.space.call_args(method, Arguments(self.space, args_w))
 
     def getitem_impl(self, name, args_w):
         space = self.space
@@ -771,25 +817,25 @@
             if c_fullname != fullname:
                 self.master.overloads[c_fullname] = method
 
-        return method.descr_get(self.w_this, [])
+        return method.descr_get(self.w_this, None)
 
 
 class W_CPPTemplateOverload(W_CPPOverload, TemplateOverloadMixin):
     """App-level dispatcher to allow both lookup/instantiation of templated methods and
     dispatch among overloads between templated and non-templated method."""
 
-    _attrs_ = ['name', 'overloads', 'master']
+    _attrs_ = ['name', 'overloads', 'master', 'w_this']
     _immutable_fields_ = ['name']
 
-    def __init__(self, space, name, declaring_scope, functions, flags = OVERLOAD_FLAGS_USE_FFI):
-         W_CPPOverload.__init__(self, space, declaring_scope, functions, flags)
+    def __init__(self, space, name, decl_scope, functions, flags = OVERLOAD_FLAGS_USE_FFI):
+         W_CPPOverload.__init__(self, space, decl_scope, functions, flags)
          self.name = name
          self.overloads = {}
          self.master = self
+         self.w_this = space.w_None
 
-    @unwrap_spec(args_w='args_w')
-    def descr_get(self, w_cppinstance, args_w):
-        # like W_CPPOverload, but returns W_CPPTemplateOverload
+    def descr_get(self, w_cppinstance, w_cls=None):
+        # TODO: don't return copy, but bind in an external object (like W_CPPOverload)
         if self.space.is_w(w_cppinstance, self.space.w_None):
             return self  # unbound, so no new instance needed
         cppol = W_CPPTemplateOverload(self.space, self.name, self.scope, self.functions, self.flags)
@@ -798,13 +844,13 @@
         return cppol     # bound
 
     @unwrap_spec(args_w='args_w')
-    def call(self, args_w):
+    def call_args(self, args_w):
         # direct call: either pick non-templated overload or attempt to deduce
         # the template instantiation from the argument types
 
         # try existing overloads or compile-time instantiations
         try:
-            return W_CPPOverload.call(self, args_w)
+            return W_CPPOverload.call_args(self, args_w)
         except Exception:
             pass
 
@@ -814,34 +860,37 @@
     def getitem(self, args_w):
         return self.getitem_impl(self.name, args_w)
 
+    def getname(self, space):
+        return self.name.decode('latin-1')
+
     def __repr__(self):
         return "W_CPPTemplateOverload(%s)" % [f.prototype() for f in self.functions]
 
 W_CPPTemplateOverload.typedef = TypeDef(
     'CPPTemplateOverload',
-    __get__     = interp2app(W_CPPTemplateOverload.descr_get),
-    __getitem__ = interp2app(W_CPPTemplateOverload.getitem),
-    __call__    = interp2app(W_CPPTemplateOverload.call),
-    __useffi__  = GetSetProperty(W_CPPTemplateOverload.fget_useffi, W_CPPTemplateOverload.fset_useffi),
-    __doc__     = GetSetProperty(W_CPPTemplateOverload.fget_doc)
+    __get__      = interp2app(W_CPPTemplateOverload.descr_get),
+    __getitem__  = interp2app(W_CPPTemplateOverload.getitem),
+    __call__     = interp2app(W_CPPTemplateOverload.call_args),
+    __useffi__   = GetSetProperty(W_CPPTemplateOverload.fget_useffi, W_CPPTemplateOverload.fset_useffi),
+    __doc__      = GetSetProperty(W_CPPTemplateOverload.fget_doc)
 )
 
 class W_CPPTemplateStaticOverload(W_CPPStaticOverload, TemplateOverloadMixin):
     """App-level dispatcher to allow both lookup/instantiation of templated methods and
     dispatch among overloads between templated and non-templated method."""
 
-    _attrs_ = ['name', 'overloads', 'master']
+    _attrs_ = ['name', 'overloads', 'master', 'w_this']
     _immutable_fields_ = ['name']
 
-    def __init__(self, space, name, declaring_scope, functions, flags = OVERLOAD_FLAGS_USE_FFI):
-         W_CPPStaticOverload.__init__(self, space, declaring_scope, functions, flags)
+    def __init__(self, space, name, decl_scope, funcs, flags = OVERLOAD_FLAGS_USE_FFI):
+         W_CPPStaticOverload.__init__(self, space, decl_scope, funcs, flags)
          self.name = name
          self.overloads = {}
          self.master = self
+         self.w_this = space.w_None
 
-    @unwrap_spec(args_w='args_w')
-    def descr_get(self, w_cppinstance, args_w):
-        # like W_CPPStaticOverload, but returns W_CPPTemplateStaticOverload
+    def descr_get(self, w_cppinstance, w_cls=None):
+        # TODO: don't return copy, but bind in an external object (like W_CPPOverload)
         if isinstance(w_cppinstance, W_CPPInstance):
             cppinstance = self.space.interp_w(W_CPPInstance, w_cppinstance)
             if cppinstance.clsdecl.handle != self.scope.handle:
@@ -852,13 +901,13 @@
         return self      # unbound
 
     @unwrap_spec(args_w='args_w')
-    def call(self, args_w):
+    def call_args(self, args_w):
         # direct call: either pick non-templated overload or attempt to deduce
         # the template instantiation from the argument types
 
         # try existing overloads or compile-time instantiations
         try:
-            return W_CPPStaticOverload.call(self, args_w)
+            return W_CPPStaticOverload.call_args(self, args_w)
         except Exception:
             pass
 
@@ -869,16 +918,19 @@
     def getitem(self, args_w):
         return self.getitem_impl(self.name, args_w)
 
+    def getname(self, space):
+        return self.name.decode('latin-1')
+
     def __repr__(self):
         return "W_CPPTemplateStaticOverload(%s)" % [f.prototype() for f in self.functions]
 
 W_CPPTemplateStaticOverload.typedef = TypeDef(
     'CPPTemplateStaticOverload',
-    __get__     = interp2app(W_CPPTemplateStaticOverload.descr_get),
-    __getitem__ = interp2app(W_CPPTemplateStaticOverload.getitem),
-    __call__    = interp2app(W_CPPTemplateStaticOverload.call),
-    __useffi__  = GetSetProperty(W_CPPTemplateStaticOverload.fget_useffi, W_CPPTemplateStaticOverload.fset_useffi),
-    __doc__     = GetSetProperty(W_CPPTemplateStaticOverload.fget_doc)
+    __get__      = interp2app(W_CPPTemplateStaticOverload.descr_get),
+    __getitem__  = interp2app(W_CPPTemplateStaticOverload.getitem),
+    __call__     = interp2app(W_CPPTemplateStaticOverload.call_args),
+    __useffi__   = GetSetProperty(W_CPPTemplateStaticOverload.fget_useffi, W_CPPTemplateStaticOverload.fset_useffi),
+    __doc__      = GetSetProperty(W_CPPTemplateStaticOverload.fget_doc)
 )
 
 
@@ -898,14 +950,15 @@
     _attrs_ = ['space', 'scope', 'converter', 'offset']
     _immutable_fields = ['scope', 'converter', 'offset']
 
-    def __init__(self, space, declaring_scope, type_name, offset):
+    def __init__(self, space, decl_scope, type_name, offset):
         self.space = space
-        self.scope = declaring_scope
+        self.scope = decl_scope
         self.converter = converter.get_converter(self.space, type_name, '')
         self.offset = offset
 
     def _get_offset(self, cppinstance):
         if cppinstance:
+            assert isinstance(cppinstance.clsdecl, W_CPPClassDecl)
             assert lltype.typeOf(cppinstance.clsdecl.handle) == lltype.typeOf(self.scope.handle)
             offset = self.offset + cppinstance.clsdecl.get_base_offset(cppinstance, self.scope)
         else:
@@ -1051,6 +1104,8 @@
         sig = '(%s)' % signature
         for f in overload.functions:
             if f.signature(False) == sig:
+                if isinstance(overload, W_CPPStaticOverload):
+                    return W_CPPStaticOverload(self.space, self, [f])
                 return W_CPPOverload(self.space, self, [f])
         raise oefmt(self.space.w_LookupError, "no overload matches signature")
 
@@ -1146,9 +1201,13 @@
 
 
 class W_CPPClassDecl(W_CPPScopeDecl):
-    _attrs_ = ['space', 'handle', 'name', 'overloads', 'datamembers']
+    _attrs_ = ['space', 'handle', 'name', 'overloads', 'datamembers', 'cppobjects']
     _immutable_fields_ = ['handle', 'name', 'overloads[*]', 'datamembers[*]']
 
+    def __init__(self, space, opaque_handle, final_scoped_name):
+        W_CPPScopeDecl.__init__(self, space, opaque_handle, final_scoped_name)
+        self.cppobjects = rweakref.RWeakValueDictionary(int, W_CPPInstance)
+
     def _build_overloads(self):
         assert len(self.overloads) == 0
         methods_tmp = {}; ftype_tmp = {}
@@ -1187,7 +1246,10 @@
             ftype = ftype_tmp[pyname]
             CPPMethodSort(methods).sort()
             if ftype & FUNCTION_IS_CONSTRUCTOR:
-                overload = W_CPPConstructorOverload(self.space, self, methods[:])
+                if capi.c_is_abstract(self.space, self.handle):
+                    overload = W_CPPAbstractCtorOverload(self.space, self, methods[:])
+                else:
+                    overload = W_CPPConstructorOverload(self.space, self, methods[:])
             elif ftype & FUNCTION_IS_STATIC:
                 if ftype & FUNCTION_IS_TEMPLATE:
                     cppname = capi.c_method_name(self.space, methods[0].cppmethod)
@@ -1253,10 +1315,12 @@
         raise self.missing_attribute_error(name)
 
     def get_base_offset(self, cppinstance, calling_scope):
+        assert isinstance(cppinstance.clsdecl, W_CPPClassDecl)
         assert self == cppinstance.clsdecl
         return 0
 
     def get_cppthis(self, cppinstance, calling_scope):
+        assert isinstance(cppinstance.clsdecl, W_CPPClassDecl)
         assert self == cppinstance.clsdecl
         return cppinstance.get_rawobject()
 
@@ -1292,12 +1356,14 @@
 
 class W_CPPComplexClassDecl(W_CPPClassDecl):
     def get_base_offset(self, cppinstance, calling_scope):
+        assert isinstance(cppinstance.clsdecl, W_CPPComplexClassDecl)
         assert self == cppinstance.clsdecl
         offset = capi.c_base_offset(self.space,
-                                    self, calling_scope, cppinstance.get_rawobject(), 1)
+            self, calling_scope, cppinstance.get_rawobject(), 1)
         return offset
 
     def get_cppthis(self, cppinstance, calling_scope):
+        assert isinstance(cppinstance.clsdecl, W_CPPComplexClassDecl)
         assert self == cppinstance.clsdecl
         offset = self.get_base_offset(cppinstance, calling_scope)
         return capi.direct_ptradd(cppinstance.get_rawobject(), offset)
@@ -1317,9 +1383,9 @@
 
 
 class W_CPPInstance(W_Root):
-    _attrs_ = ['space', 'clsdecl', '_rawobject', 'smartdecl', 'deref', 'flags',
+    _attrs_ = ['space', 'clsdecl', '_rawobject', 'smartdecl', 'deref', 'flags', 'rt_flags',
                'finalizer_registered']
-    _immutable_fields_ = ['clsdecl', 'smartdecl', 'deref']
+    _immutable_fields_ = ['clsdecl', 'smartdecl', 'deref', 'flags']
 
     finalizer_registered = False
 
@@ -1327,6 +1393,7 @@
                  smartdecl=None, deref=rffi.cast(capi.C_METHOD, 0)):
         self.space = space
         self.clsdecl = decl
+        assert isinstance(self.clsdecl, W_CPPClassDecl)
         assert lltype.typeOf(rawobject) == capi.C_OBJECT
         assert not isref or rawobject
         self._rawobject = rawobject
@@ -1334,15 +1401,16 @@
         self.flags = 0
         if isref or (smartdecl and deref):
             self.flags |= INSTANCE_FLAGS_IS_REF
+        self.rt_flags = 0
         if python_owns:
-            self.flags |= INSTANCE_FLAGS_PYTHON_OWNS
+            self.rt_flags |= INSTANCE_FLAGS_PYTHON_OWNS
             self._opt_register_finalizer()
         self.smartdecl = smartdecl
         self.deref     = deref
 
     def _opt_register_finalizer(self):
         if not self.finalizer_registered and not hasattr(self.space, "fake"):
-            assert self.flags & INSTANCE_FLAGS_PYTHON_OWNS
+            assert self.rt_flags & INSTANCE_FLAGS_PYTHON_OWNS
             self.register_finalizer(self.space)
             self.finalizer_registered = True
 

From pypy.commits at gmail.com  Sat Jul 28 05:01:42 2018
From: pypy.commits at gmail.com (wlav)
Date: Sat, 28 Jul 2018 02:01:42 -0700 (PDT)
Subject: [pypy-commit] pypy cppyy-packaging: restore std::vector speedups
Message-ID: <5b5c30f6.1c69fb81.128a5.ca33@mx.google.com>

Author: Wim Lavrijsen <WLavrijsen at lbl.gov>
Branch: cppyy-packaging
Changeset: r94910:422177baf4ea
Date: 2018-07-28 01:15 -0700
http://bitbucket.org/pypy/pypy/changeset/422177baf4ea/

Log:	restore std::vector speedups

diff --git a/pypy/module/_cppyy/capi/loadable_capi.py b/pypy/module/_cppyy/capi/loadable_capi.py
--- a/pypy/module/_cppyy/capi/loadable_capi.py
+++ b/pypy/module/_cppyy/capi/loadable_capi.py
@@ -1,13 +1,18 @@
 import os
+
 from rpython.rtyper.lltypesystem import rffi, lltype
 from rpython.rlib.rarithmetic import intmask
 from rpython.rlib import jit, jit_libffi, libffi, rdynload, objectmodel
 from rpython.rlib.rarithmetic import r_singlefloat
 from rpython.tool import leakfinder
 
-from pypy.interpreter.gateway import interp2app
-from pypy.interpreter.error import oefmt
+from pypy.interpreter.error import OperationError, oefmt
+from pypy.interpreter.argument import Arguments
+from pypy.interpreter.gateway import interp2app, interpindirect2app
+from pypy.interpreter.typedef import TypeDef
+from pypy.objspace.std.iterobject import W_AbstractSeqIterObject
 
+from pypy.module._rawffi.array import W_ArrayInstance
 from pypy.module._cffi_backend import ctypefunc, ctypeprim, cdataobj, misc
 from pypy.module._cffi_backend import newtype
 from pypy.module._cppyy import ffitypes
@@ -274,8 +279,9 @@
             'stdstring2charp'          : ([c_object, c_voidp],        c_ccharp),
             'stdstring2stdstring'      : ([c_object],                 c_object),
 
-            'stdvector_valuetype'      : ([c_ccharp],                 c_ccharp),
-            'stdvector_valuesize'      : ([c_ccharp],                 c_size_t),
+            'longdouble2double'        : ([c_voidp],                  c_double),
+            'double2longdouble'        : ([c_double, c_voidp],        c_void),
+
             'vectorbool_getitem'       : ([c_object, c_int],          c_int),
             'vectorbool_setitem'       : ([c_object, c_int, c_int],   c_void),
         }
@@ -658,13 +664,6 @@
 def c_stdstring2stdstring(space, cppobject):
     return _cdata_to_cobject(space, call_capi(space, 'stdstring2stdstring', [_ArgH(cppobject)]))
 
-def c_stdvector_valuetype(space, pystr):
-    return charp2str_free(space, call_capi(space, 'stdvector_valuetype', [_ArgS(pystr)]))
-
-def c_stdvector_valuetype(space, pystr):
-    return charp2str_free(space, call_capi(space, 'stdvector_valuetype', [_ArgS(pystr)]))
-def c_stdvector_valuesize(space, pystr):
-    return _cdata_to_size_t(space, call_capi(space, 'stdvector_valuesize', [_ArgS(pystr)]))
 def c_vectorbool_getitem(space, vbool, idx):
     return call_capi(space, 'vectorbool_getitem', [_ArgH(vbool), _ArgL(idx)])
 def c_vectorbool_setitem(space, vbool, idx, value):
@@ -702,6 +701,52 @@
     idx = vbool_getindex(space, w_self, w_idx)
     c_vectorbool_setitem(space, vbool._rawobject, idx, int(space.is_true(w_value)))
 
+class W_STLVectorIter(W_AbstractSeqIterObject):
+    # w_seq and index are in base class
+    _immutable_fields_ = ['converter', 'data', 'len', 'stride']
+
+    def __init__(self, space, w_vector):
+        W_AbstractSeqIterObject.__init__(self, w_vector)
+        # TODO: this should live in rpythonize.py or something so that the
+        # imports can move to the top w/o getting circles
+        from pypy.module._cppyy import interp_cppyy
+        assert isinstance(w_vector, interp_cppyy.W_CPPInstance)
+        vector = space.interp_w(interp_cppyy.W_CPPInstance, w_vector)
+
+        v_type = c_resolve_name(space, vector.clsdecl.name+'::value_type')
+        v_size = c_size_of_type(space, v_type)
+
+        if not v_type or not v_size:
+            raise NotImplementedError   # fallback on getitem
+
+        from pypy.module._cppyy import converter
+        self.converter = converter.get_converter(space, v_type, '')
+
+        # this 'data' is from the decl, so not the pythonized data from pythonify.py
+        w_arr = space.call_obj_args(vector.clsdecl.get_overload('data'), w_vector, Arguments(space, []))
+        arr = space.interp_w(W_ArrayInstance, w_arr, can_be_None=True)
+        if not arr:
+            raise OperationError(space.w_StopIteration, space.w_None)
+
+        self.data    = rffi.cast(rffi.CCHARP, space.uint_w(arr.getbuffer(space)))
+        self.len     = space.uint_w(space.call_obj_args(vector.clsdecl.get_overload('size'), w_vector, Arguments(space, [])))
+        self.stride  = v_size
+
+    def descr_next(self, space):
+        if self.w_seq is None:
+            raise OperationError(space.w_StopIteration, space.w_None)
+        if self.len <= self.index:
+            self.w_seq = None
+            raise OperationError(space.w_StopIteration, space.w_None)
+        offset = lltype.direct_ptradd(self.data, rffi.cast(rffi.SIZE_T, self.index*self.stride))
+        w_item = self.converter.from_memory(space, space.w_None, rffi.cast(rffi.LONG, offset))
+        self.index += 1
+        return w_item
+
+def stdvector_iter(space, w_self):
+    return W_STLVectorIter(space, w_self)
+
+
 # setup pythonizations for later use at run-time
 _pythonizations = {}
 def register_pythonizations(space):
@@ -712,6 +757,9 @@
         ### std::string
         stdstring_c_str,
 
+        ### std::vector
+        stdvector_iter,
+
         ### std::vector<bool>
         vectorbool_getitem,
         vectorbool_setitem,
@@ -730,6 +778,9 @@
         _method_alias(space, w_pycppclass, "_cppyy_as_builtin", "c_str")
         _method_alias(space, w_pycppclass, "__str__",           "c_str")
 
-    if name == "std::vector<bool>":
+    if  name.find("std::vector<bool", 0, 16) == 0:
         space.setattr(w_pycppclass, space.newtext("__getitem__"), _pythonizations["vectorbool_getitem"])
         space.setattr(w_pycppclass, space.newtext("__setitem__"), _pythonizations["vectorbool_setitem"])
+
+    elif name.find("std::vector", 0, 11) == 0:
+        space.setattr(w_pycppclass, space.newtext("__iter__"), _pythonizations["stdvector_iter"])
diff --git a/pypy/module/_cppyy/include/capi.h b/pypy/module/_cppyy/include/capi.h
--- a/pypy/module/_cppyy/include/capi.h
+++ b/pypy/module/_cppyy/include/capi.h
@@ -220,10 +220,6 @@
     cppyy_object_t cppyy_stdstring2stdstring(cppyy_object_t ptr);
 
     RPY_EXTERN
-    const char* cppyy_stdvector_valuetype(const char* clname);
-    RPY_EXTERN
-    size_t      cppyy_stdvector_valuesize(const char* clname);
-    RPY_EXTERN
     int         cppyy_vectorbool_getitem(cppyy_object_t ptr, int idx);
     RPY_EXTERN
     void        cppyy_vectorbool_setitem(cppyy_object_t ptr, int idx, int value);
diff --git a/pypy/module/_cppyy/pythonify.py b/pypy/module/_cppyy/pythonify.py
--- a/pypy/module/_cppyy/pythonify.py
+++ b/pypy/module/_cppyy/pythonify.py
@@ -435,9 +435,8 @@
     # map begin()/end() protocol to iter protocol on STL(-like) classes, but
     # not on vector, which is pythonized in the capi (interp-level; there is
     # also the fallback on the indexed __getitem__, but that is slower)
-# TODO:    if not (0 <= name.find('vector') <= 5):
     add_checked_item = False
-    if not (0 <= name.find('vector<bool') <= 5):
+    if name.find('std::vector', 0, 11) != 0:
         if ('begin' in pyclass.__dict__ and 'end' in pyclass.__dict__):
             if _cppyy._scope_byname(name+'::iterator') or \
                     _cppyy._scope_byname(name+'::const_iterator'):
@@ -454,7 +453,7 @@
                 add_checked_item = True
 
     # add python collection based initializer
-    if 0 <= name.find('vector') <= 5:
+    if name.find('std::vector', 0, 11) == 0:
         pyclass.__real_init__ = pyclass.__init__
         def vector_init(self, *args):
             if len(args) == 1 and isinstance(args[0], (tuple, list)):
@@ -478,8 +477,8 @@
 
     # TODO: must be a simpler way to check (or at least hook these to a namespace
     # std specific pythonizor)
-    if add_checked_item or 0 <= name.find('vector') <= 5 or \
-            0 <= name.find('array') <= 5 or 0 <= name.find('deque') <= 5:
+    if add_checked_item or name.find('std::vector', 0, 11) == 0 or \
+            name.find('std::array', 0, 11) == 0 or name.find('std::deque', 0, 10) == 0:
         # combine __getitem__ and __len__ to make a pythonized __getitem__
         if '__getitem__' in pyclass.__dict__ and '__len__' in pyclass.__dict__:
             pyclass._getitem__unchecked = pyclass.__getitem__

From pypy.commits at gmail.com  Sat Jul 28 07:29:39 2018
From: pypy.commits at gmail.com (nurbldoff)
Date: Sat, 28 Jul 2018 04:29:39 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6: Add missing argument to scandir.
Message-ID: <5b5c53a3.1c69fb81.e9df3.6e58@mx.google.com>

Author: Johan Forsberg <johan at slentrian.org>
Branch: py3.6
Changeset: r94911:11f17afc7367
Date: 2018-07-28 12:24 +0100
http://bitbucket.org/pypy/pypy/changeset/11f17afc7367/

Log:	Add missing argument to scandir.

diff --git a/pypy/module/posix/test/test_scandir.py b/pypy/module/posix/test/test_scandir.py
--- a/pypy/module/posix/test/test_scandir.py
+++ b/pypy/module/posix/test/test_scandir.py
@@ -224,7 +224,7 @@
 
     def test_lstat(self):
         posix = self.posix
-        d = next(posix.scandir())
+        d = next(posix.scandir(self.dir1))
         with open(d) as fp:
             length = len(fp.read())
         assert posix.lstat(d).st_size == length

From pypy.commits at gmail.com  Sat Jul 28 08:07:47 2018
From: pypy.commits at gmail.com (cfbolz)
Date: Sat, 28 Jul 2018 05:07:47 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: tweak
Message-ID: <5b5c5c93.1c69fb81.f96cc.c88c@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: py3.5
Changeset: r94912:18f7d184a9f7
Date: 2018-07-28 14:02 +0200
http://bitbucket.org/pypy/pypy/changeset/18f7d184a9f7/

Log:	tweak

diff --git a/pypy/interpreter/astcompiler/codegen.py b/pypy/interpreter/astcompiler/codegen.py
--- a/pypy/interpreter/astcompiler/codegen.py
+++ b/pypy/interpreter/astcompiler/codegen.py
@@ -367,8 +367,8 @@
                                        names)
         self._visit_arg_annotations(args.kwonlyargs, names)
         kwarg = args.kwarg
-        if args.kwarg:
-            self._visit_arg_annotation(args.kwarg.arg, args.kwarg.annotation,
+        if kwarg:
+            self._visit_arg_annotation(kwarg.arg, kwarg.annotation,
                                        names)
         self._visit_arg_annotation("return", returns, names)
         l = len(names)

From pypy.commits at gmail.com  Sat Jul 28 08:07:49 2018
From: pypy.commits at gmail.com (cfbolz)
Date: Sat, 28 Jul 2018 05:07:49 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: fix annotation bug reported by hubo on
 pypy-dev
Message-ID: <5b5c5c95.1c69fb81.a1e70.1b1b@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: py3.5
Changeset: r94913:179c172169f1
Date: 2018-07-28 14:06 +0200
http://bitbucket.org/pypy/pypy/changeset/179c172169f1/

Log:	fix annotation bug reported by hubo on pypy-dev

diff --git a/pypy/interpreter/astcompiler/symtable.py b/pypy/interpreter/astcompiler/symtable.py
--- a/pypy/interpreter/astcompiler/symtable.py
+++ b/pypy/interpreter/astcompiler/symtable.py
@@ -622,6 +622,10 @@
         assert isinstance(args, ast.arguments)
         if args.args:
             self._visit_arg_annotations(args.args)
+        if args.vararg:
+            self._visit_arg_annotation(args.vararg)
+        if args.kwarg:
+            self._visit_arg_annotation(args.kwarg)
         if args.kwonlyargs:
             self._visit_arg_annotations(args.kwonlyargs)
         if func.returns:
@@ -630,8 +634,11 @@
     def _visit_arg_annotations(self, args):
         for arg in args:
             assert isinstance(arg, ast.arg)
-            if arg.annotation:
-                arg.annotation.walkabout(self)
+            self._visit_arg_annotation(arg)
+
+    def _visit_arg_annotation(self, arg):
+        if arg.annotation:
+            arg.annotation.walkabout(self)
 
     def visit_Name(self, name):
         if name.ctx == ast.Load:
diff --git a/pypy/interpreter/test/test_syntax.py b/pypy/interpreter/test/test_syntax.py
--- a/pypy/interpreter/test/test_syntax.py
+++ b/pypy/interpreter/test/test_syntax.py
@@ -691,6 +691,16 @@
                                     "bye" : 5, "kw" : 6, "return" : 42}
         """
 
+    def test_bug_annotations_lambda(self):
+        """
+        # those used to crash
+        def broken(*a: lambda x: None):
+            pass
+
+        def broken(**a: lambda x: None):
+            pass
+        """
+
 class AppTestSyntaxError:
 
     def test_tokenizer_error_location(self):

From pypy.commits at gmail.com  Sat Jul 28 09:23:52 2018
From: pypy.commits at gmail.com (cfbolz)
Date: Sat, 28 Jul 2018 06:23:52 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6: merge py3.6-wordcode
Message-ID: <5b5c6e68.1c69fb81.e75e2.0bd1@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: py3.6
Changeset: r94914:2e5eb984c7fa
Date: 2018-07-28 15:22 +0200
http://bitbucket.org/pypy/pypy/changeset/2e5eb984c7fa/

Log:	merge py3.6-wordcode

	switch the py3.6 over to the new wordcode format introduced in
	CPython 3.6.

diff too long, truncating to 2000 out of 2009 lines

diff --git a/lib-python/3/opcode.py b/lib-python/3/opcode.py
--- a/lib-python/3/opcode.py
+++ b/lib-python/3/opcode.py
@@ -216,6 +216,7 @@
 def_op('BUILD_SET_UNPACK', 153)
 
 def_op('FORMAT_VALUE', 155)   # in CPython 3.6, but available in PyPy from 3.5
+def_op('BUILD_CONST_KEY_MAP', 156)
 def_op('BUILD_STRING', 157)   # in CPython 3.6, but available in PyPy from 3.5
 
 # pypy modification, experimental bytecode
diff --git a/lib-python/3/test/test_opcodes.py b/lib-python/3/test/test_opcodes.py
--- a/lib-python/3/test/test_opcodes.py
+++ b/lib-python/3/test/test_opcodes.py
@@ -27,7 +27,9 @@
             with open(ann_module.__file__) as f:
                 txt = f.read()
             co = compile(txt, ann_module.__file__, 'exec')
-            self.assertEqual(co.co_firstlineno, 6)
+            # On PyPy, the lineno of multiline tokens is the *first* line, on
+            # CPython the last (CPython expects 6 here)
+            self.assertEqual(co.co_firstlineno, 3)
         except OSError:
             pass
 
diff --git a/pypy/interpreter/astcompiler/assemble.py b/pypy/interpreter/astcompiler/assemble.py
--- a/pypy/interpreter/astcompiler/assemble.py
+++ b/pypy/interpreter/astcompiler/assemble.py
@@ -21,6 +21,8 @@
     def __init__(self, opcode, arg=0):
         self.opcode = opcode
         self.arg = arg
+        if opcode < ops.HAVE_ARGUMENT:
+            assert arg == 0
         self.lineno = 0
         self.has_jump = False
 
@@ -28,9 +30,33 @@
         """Return the size of bytes of this instruction when it is
         encoded.
         """
-        if self.opcode >= ops.HAVE_ARGUMENT:
-            return (6 if self.arg > 0xFFFF else 3)
-        return 1
+        if self.arg <= 0xff:
+            return 2
+        if self.arg <= 0xffff:
+            return 4
+        if self.arg <= 0xffffff:
+            return 6
+        return 8
+
+    def encode(self, code):
+        opcode = self.opcode
+
+        arg = self.arg
+        size = self.size()
+        if size == 8:
+            code.append(chr(ops.EXTENDED_ARG))
+            code.append(chr((arg >> 24) & 0xff))
+            assert ((arg >> 24) & 0xff) == (arg >> 24)
+        if size >= 6:
+            code.append(chr(ops.EXTENDED_ARG))
+            code.append(chr((arg >> 16) & 0xff))
+        if size >= 4:
+            code.append(chr(ops.EXTENDED_ARG))
+            code.append(chr((arg >> 8) & 0xff))
+        if size >= 2:
+            code.append(chr(opcode))
+            code.append(chr(arg & 0xff))
+
 
     def jump_to(self, target, absolute=False):
         """Indicate the target this jump instruction.
@@ -121,20 +147,9 @@
         """Encode the instructions in this block into bytecode."""
         code = []
         for instr in self.instructions:
-            opcode = instr.opcode
-            if opcode >= ops.HAVE_ARGUMENT:
-                arg = instr.arg
-                if instr.arg > 0xFFFF:
-                    ext = arg >> 16
-                    code.append(chr(ops.EXTENDED_ARG))
-                    code.append(chr(ext & 0xFF))
-                    code.append(chr(ext >> 8))
-                    arg &= 0xFFFF
-                code.append(chr(opcode))
-                code.append(chr(arg & 0xFF))
-                code.append(chr(arg >> 8))
-            else:
-                code.append(chr(opcode))
+            instr.encode(code)
+        assert len(code) == self.code_size()
+        assert len(code) & 1 == 0
         return ''.join(code)
 
 
@@ -185,6 +200,13 @@
         self.lineno = 0
         self.add_none_to_final_return = True
 
+    def _check_consistency(self, blocks):
+        current_off = 0
+        for block in blocks:
+            assert block.offset == current_off
+            for instr in block.instructions:
+                current_off += instr.size()
+
     def new_block(self):
         return Block()
 
@@ -315,14 +337,12 @@
 
     def _resolve_block_targets(self, blocks):
         """Compute the arguments of jump instructions."""
-        last_extended_arg_count = 0
         # The reason for this loop is extended jumps.  EXTENDED_ARG
-        # extends the bytecode size, so it might invalidate the offsets
-        # we've already given.  Thus we have to loop until the number of
-        # extended args is stable.  Any extended jump at all is
-        # extremely rare, so performance is not too concerning.
+        # extends the bytecode size, so it might invalidate the offsets we've
+        # already given.  Thus we have to loop until the size of all jump
+        # instructions is stable. Any extended jump at all is extremely rare,
+        # so performance should not be too concerning.
         while True:
-            extended_arg_count = 0
             offset = 0
             force_redo = False
             # Calculate the code offset of each block.
@@ -332,7 +352,8 @@
             for block in blocks:
                 offset = block.offset
                 for instr in block.instructions:
-                    offset += instr.size()
+                    size = instr.size()
+                    offset += size
                     if instr.has_jump:
                         target, absolute = instr.jump
                         op = instr.opcode
@@ -351,22 +372,21 @@
                                     instr.opcode = ops.RETURN_VALUE
                                     instr.arg = 0
                                     instr.has_jump = False
-                                    # The size of the code changed,
+                                    # The size of the code maybe have changed,
                                     # we have to trigger another pass
-                                    force_redo = True
+                                    if instr.size() != size:
+                                        force_redo = True
                                     continue
                         if absolute:
                             jump_arg = target.offset
                         else:
                             jump_arg = target.offset - offset
                         instr.arg = jump_arg
-                        if jump_arg > 0xFFFF:
-                            extended_arg_count += 1
-            if (extended_arg_count == last_extended_arg_count and
-                not force_redo):
-                break
-            else:
-                last_extended_arg_count = extended_arg_count
+                        if instr.size() != size:
+                            force_redo = True
+            if not force_redo:
+                self._check_consistency(blocks)
+                return
 
     def _build_consts_array(self):
         """Turn the applevel constants dictionary into a list."""
@@ -738,7 +758,7 @@
     return -2 - _num_args(arg) - ((arg >> 16) & 0xFFFF)
 
 def _compute_MAKE_FUNCTION(arg):
-    return -1 - _num_args(arg) - ((arg >> 16) & 0xFFFF)
+    return -1 - bool(arg & 0x01) - bool(arg & 0x02) - bool(arg & 0x04) - bool(arg & 0x08)
 
 def _compute_BUILD_SLICE(arg):
     if arg == 3:
@@ -775,6 +795,9 @@
 def _compute_BUILD_STRING(arg):
     return 1 - arg
 
+def _compute_BUILD_CONST_KEY_MAP(arg):
+    return  -arg
+
 
 _stack_effect_computers = {}
 for name, func in globals().items():
diff --git a/pypy/interpreter/astcompiler/codegen.py b/pypy/interpreter/astcompiler/codegen.py
--- a/pypy/interpreter/astcompiler/codegen.py
+++ b/pypy/interpreter/astcompiler/codegen.py
@@ -307,7 +307,7 @@
         # if the scope contained an annotated variable assignemt,
         # this will emit the requisite SETUP_ANNOTATIONS
         if self.scope.contains_annotated and not isinstance(self, AbstractFunctionCodeGenerator):
-            self.emit_op(ops.SETUP_ANNOTATIONS)
+            return self.emit_op(ops.SETUP_ANNOTATIONS)
 
     def visit_Module(self, mod):
         if not self._handle_body(mod.body):
@@ -321,10 +321,11 @@
         self.add_none_to_final_return = False
         mod.body.walkabout(self)
 
-    def _make_function(self, code, num_defaults=0, qualname=None):
+    def _make_function(self, code, oparg=0, qualname=None):
         """Emit the opcodes to turn a code object into a function."""
         w_qualname = self.space.newtext(qualname or code.co_name)
         if code.co_freevars:
+            oparg = oparg | 0x08
             # Load cell and free vars to pass on.
             for free in code.co_freevars:
                 free_scope = self.scope.lookup(free)
@@ -335,24 +336,25 @@
                     index = self.free_vars[free]
                 self.emit_op_arg(ops.LOAD_CLOSURE, index)
             self.emit_op_arg(ops.BUILD_TUPLE, len(code.co_freevars))
-            self.load_const(code)
-            self.load_const(w_qualname)
-            self.emit_op_arg(ops.MAKE_CLOSURE, num_defaults)
-        else:
-            self.load_const(code)
-            self.load_const(w_qualname)
-            self.emit_op_arg(ops.MAKE_FUNCTION, num_defaults)
+        self.load_const(code)
+        self.load_const(w_qualname)
+        self.emit_op_arg(ops.MAKE_FUNCTION, oparg)
 
     def _visit_kwonlydefaults(self, args):
         defaults = 0
+        keys_w = []
         for i, default in enumerate(args.kw_defaults):
             if default:
                 kwonly = args.kwonlyargs[i]
                 assert isinstance(kwonly, ast.arg)
                 mangled = self.scope.mangle(kwonly.arg)
-                self.load_const(self.space.newtext(mangled))
+                keys_w.append(self.space.newtext(mangled))
                 default.walkabout(self)
                 defaults += 1
+        if keys_w:
+            w_tup = self.space.newtuple(keys_w)
+            self.load_const(w_tup)
+            self.emit_op_arg(ops.BUILD_CONST_KEY_MAP, len(keys_w))
         return defaults
 
     def _visit_arg_annotation(self, name, ann, names):
@@ -387,7 +389,7 @@
                 self.error("too many annotations", func)
             w_tup = space.newtuple([space.newtext(name) for name in names])
             self.load_const(w_tup)
-            l += 1
+            self.emit_op_arg(ops.BUILD_CONST_KEY_MAP, l)
         return l
 
     @specialize.arg(2)
@@ -396,16 +398,25 @@
         # Load decorators first, but apply them after the function is created.
         self.visit_sequence(func.decorator_list)
         args = func.args
+
         assert isinstance(args, ast.arguments)
+
+        oparg = 0
         self.visit_sequence(args.defaults)
-        kw_default_count = 0
+
+        if args.defaults is not None and len(args.defaults):
+            oparg = oparg | 0x01
+            self.emit_op_arg(ops.BUILD_TUPLE, len(args.defaults))
+
         if args.kwonlyargs:
             kw_default_count = self._visit_kwonlydefaults(args)
+            if kw_default_count:
+                oparg = oparg | 0x02
+
         num_annotations = self._visit_annotations(func, args, func.returns)
-        num_defaults = len(args.defaults) if args.defaults is not None else 0
-        oparg = num_defaults
-        oparg |= kw_default_count << 8
-        oparg |= num_annotations << 16
+        if num_annotations:
+            oparg = oparg | 0x04
+
         code, qualname = self.sub_scope(function_code_generator, func.name,
                                         func, func.lineno)
         self._make_function(code, oparg, qualname=qualname)
@@ -425,15 +436,20 @@
         self.update_position(lam.lineno)
         args = lam.args
         assert isinstance(args, ast.arguments)
+
         self.visit_sequence(args.defaults)
-        kw_default_count = 0
+
+        oparg = 0
+        if args.defaults is not None and len(args.defaults):
+            oparg = oparg | 0x01
+            self.emit_op_arg(ops.BUILD_TUPLE, len(args.defaults))
+
         if args.kwonlyargs:
             kw_default_count = self._visit_kwonlydefaults(args)
-        default_count = len(args.defaults) if args.defaults is not None else 0
+            if kw_default_count:
+                oparg = oparg | 0x02
         code, qualname = self.sub_scope(
             LambdaCodeGenerator, "<lambda>", lam, lam.lineno)
-        oparg = default_count
-        oparg |= kw_default_count << 8
         self._make_function(code, oparg, qualname=qualname)
 
     def visit_ClassDef(self, cls):
@@ -1281,28 +1297,46 @@
         containers = 0
         elements = 0
         is_unpacking = False
+        all_constant_keys_w = None
         if d.values:
+            if len(d.keys) < 0xffff:
+                all_constant_keys_w = []
+                for key in d.keys:
+                    if key is None or key.as_constant() is None:
+                        all_constant_keys_w = None
+                        break
+                    else:
+                        all_constant_keys_w.append(key.as_constant())
             for i in range(len(d.values)):
                 key = d.keys[i]
                 is_unpacking = key is None
                 if elements == 0xFFFF or (elements and is_unpacking):
+                    assert all_constant_keys_w is None
                     self.emit_op_arg(ops.BUILD_MAP, elements)
                     containers += 1
                     elements = 0
                 if is_unpacking:
+                    assert all_constant_keys_w is None
                     d.values[i].walkabout(self)
                     containers += 1
                 else:
-                    key.walkabout(self)
+                    if not all_constant_keys_w:
+                        key.walkabout(self)
                     d.values[i].walkabout(self)
                     elements += 1
         if elements or containers == 0:
-            self.emit_op_arg(ops.BUILD_MAP, elements)
-            containers += 1
+            if all_constant_keys_w:
+                w_tup = self.space.newtuple(all_constant_keys_w)
+                self.load_const(w_tup)
+                self.emit_op_arg(ops.BUILD_CONST_KEY_MAP, elements)
+            else:
+                self.emit_op_arg(ops.BUILD_MAP, elements)
+                containers += 1
         # If there is more than one dict, they need to be merged into
         # a new dict. If there is one dict and it's an unpacking, then
         #it needs to be copied into a new dict.
         while containers > 1 or is_unpacking:
+            assert all_constant_keys_w is None
             oparg = min(containers, 255)
             self.emit_op_arg(ops.BUILD_MAP_UNPACK, oparg)
             containers -= (oparg - 1)
@@ -1699,6 +1733,12 @@
                                      symbols, compile_info, qualname=None)
 
     def _compile(self, tree):
+        if isinstance(tree, ast.Module):
+            if tree.body:
+                self.first_lineno = tree.body[0].lineno
+            else:
+                self.first_lineno = self.lineno = 1
+
         self._maybe_setup_annotations()
         tree.walkabout(self)
 
diff --git a/pypy/interpreter/astcompiler/test/test_astbuilder.py b/pypy/interpreter/astcompiler/test/test_astbuilder.py
--- a/pypy/interpreter/astcompiler/test/test_astbuilder.py
+++ b/pypy/interpreter/astcompiler/test/test_astbuilder.py
@@ -939,7 +939,7 @@
 
     def test_flufl(self):
         source = "x <> y"
-        raises(SyntaxError, self.get_ast, source)
+        py.test.raises(SyntaxError, self.get_ast, source)
         comp = self.get_first_expr(source,
                                    flags=consts.CO_FUTURE_BARRY_AS_BDFL)
         assert isinstance(comp, ast.Compare)
@@ -1167,7 +1167,7 @@
         s = self.get_first_expr("b'hi' b' implicitly' b' extra'")
         assert isinstance(s, ast.Bytes)
         assert space.eq_w(s.s, space.newbytes("hi implicitly extra"))
-        raises(SyntaxError, self.get_first_expr, "b'hello' 'world'")
+        py.test.raises(SyntaxError, self.get_first_expr, "b'hello' 'world'")
         sentence = u"Die M&#228;nner &#228;rgern sich!"
         source = u"# coding: utf-7\nstuff = '%s'" % (sentence,)
         info = pyparse.CompileInfo("<test>", "exec")
@@ -1362,8 +1362,8 @@
         assert isinstance(if2, ast.Name)
 
     def test_cpython_issue12983(self):
-        raises(SyntaxError, self.get_ast, r"""b'\x'""")
-        raises(SyntaxError, self.get_ast, r"""b'\x0'""")
+        py.test.raises(SyntaxError, self.get_ast, r"""b'\x'""")
+        py.test.raises(SyntaxError, self.get_ast, r"""b'\x0'""")
 
     def test_matmul(self):
         mod = self.get_ast("a @ b")
diff --git a/pypy/interpreter/astcompiler/test/test_compiler.py b/pypy/interpreter/astcompiler/test/test_compiler.py
--- a/pypy/interpreter/astcompiler/test/test_compiler.py
+++ b/pypy/interpreter/astcompiler/test/test_compiler.py
@@ -72,10 +72,9 @@
     def check(self, w_dict, evalexpr, expected):
         # for now, we compile evalexpr with CPython's compiler but run
         # it with our own interpreter to extract the data from w_dict
-        co_expr = compile(evalexpr, '<evalexpr>', 'eval')
         space = self.space
-        pyco_expr = PyCode._from_code(space, co_expr)
-        w_res = pyco_expr.exec_host_bytecode(w_dict, w_dict)
+        pyco_expr = space.createcompiler().compile(evalexpr, '<evalexpr>', 'eval', 0)
+        w_res = space.exec_(pyco_expr, w_dict, w_dict)
         res = space.str_w(space.repr(w_res))
         expected_repr = self.get_py3_repr(expected)
         if isinstance(expected, float):
@@ -1011,6 +1010,7 @@
         ("C4.__doc__", 'docstring'),
         ("C4.__doc__", 'docstring'),
         ("__doc__", None),])
+
     def test_remove_docstring(self, expr, result):
         source = '"module_docstring"\n' + """if 1:
         def f1():
@@ -1210,12 +1210,12 @@
         yield self.st, """z=f'{f"{0}"*3}'""", 'z', '000'
 
     def test_fstring_error(self):
-        raises(SyntaxError, self.run, "f'{}'")
-        raises(SyntaxError, self.run, "f'{   \t   }'")
-        raises(SyntaxError, self.run, "f'{5#}'")
-        raises(SyntaxError, self.run, "f'{5)#}'")
-        raises(SyntaxError, self.run, "f'''{5)\n#}'''")
-        raises(SyntaxError, self.run, "f'\\x'")
+        py.test.raises(SyntaxError, self.run, "f'{}'")
+        py.test.raises(SyntaxError, self.run, "f'{   \t   }'")
+        py.test.raises(SyntaxError, self.run, "f'{5#}'")
+        py.test.raises(SyntaxError, self.run, "f'{5)#}'")
+        py.test.raises(SyntaxError, self.run, "f'''{5)\n#}'''")
+        py.test.raises(SyntaxError, self.run, "f'\\x'")
 
     def test_fstring_encoding(self):
         src = """# -*- coding: latin-1 -*-\nz=ord(f'{"\xd8"}')\n"""
@@ -1512,3 +1512,18 @@
             del []
         """
         generate_function_code(source, self.space)
+
+    def test_make_constant_map(self):
+        source = """def f():
+            return {"A": 1, "b": 2}
+        """
+        counts = self.count_instructions(source)
+        assert ops.BUILD_MAP not in counts
+        source = """def f():
+            return {"a": 1, "b": {}, 1: {"a": x}}
+        """
+        counts = self.count_instructions(source)
+        assert counts[ops.BUILD_MAP] == 1 # the empty dict
+        assert counts[ops.BUILD_CONST_KEY_MAP] == 2
+
+
diff --git a/pypy/interpreter/astcompiler/test/test_misc.py b/pypy/interpreter/astcompiler/test/test_misc.py
--- a/pypy/interpreter/astcompiler/test/test_misc.py
+++ b/pypy/interpreter/astcompiler/test/test_misc.py
@@ -1,4 +1,5 @@
 from pypy.interpreter.astcompiler.misc import mangle
+from pypy.interpreter.astcompiler.assemble import Instruction, ops
 
 def test_mangle():
     assert mangle("foo", "Bar") == "foo"
@@ -13,6 +14,34 @@
     assert mangle("__foo", "___") == "__foo"
     assert mangle("___foo", "__Bar") == "_Bar___foo"
 
+def test_instruction_size():
+    assert Instruction(ops.POP_TOP).size() == 2
+    assert Instruction(ops.LOAD_FAST, 23).size() == 2
+    assert Instruction(ops.LOAD_FAST, 0xfff0).size() == 4
+    assert Instruction(ops.LOAD_FAST, 0x10000).size() == 6
+    assert Instruction(ops.LOAD_FAST, 0x1000000).size() == 8
+
+def test_instruction_encode():
+    c = []
+    Instruction(ops.POP_TOP).encode(c)
+    assert c == [chr(ops.POP_TOP), '\x00']
+
+    c = []
+    Instruction(ops.LOAD_FAST, 1).encode(c)
+    assert c == [chr(ops.LOAD_FAST), '\x01']
+
+    c = []
+    Instruction(ops.LOAD_FAST, 0x201).encode(c)
+    assert c == [chr(ops.EXTENDED_ARG), '\x02', chr(ops.LOAD_FAST), '\x01']
+
+    c = []
+    Instruction(ops.LOAD_FAST, 0x30201).encode(c)
+    assert c == [chr(ops.EXTENDED_ARG), '\x03', chr(ops.EXTENDED_ARG), '\x02', chr(ops.LOAD_FAST), '\x01']
+
+    c = []
+    Instruction(ops.LOAD_FAST, 0x5030201).encode(c)
+    assert c == [chr(ops.EXTENDED_ARG), '\x05', chr(ops.EXTENDED_ARG), '\x03', chr(ops.EXTENDED_ARG), '\x02', chr(ops.LOAD_FAST), '\x01']
+
 def app_test_warning_to_error_translation():
     import warnings
 
diff --git a/pypy/interpreter/generator.py b/pypy/interpreter/generator.py
--- a/pypy/interpreter/generator.py
+++ b/pypy/interpreter/generator.py
@@ -166,16 +166,20 @@
             # Normal case: the call above raises Yield.
             # We reach this point if the iterable is exhausted.
             last_instr = jit.promote(frame.last_instr)
+            assert last_instr & 1 == 0
             assert last_instr >= 0
-            return r_uint(last_instr + 1)
+            return r_uint(last_instr + 2)
 
         if isinstance(w_arg_or_err, SApplicationException):
             return frame.handle_generator_error(w_arg_or_err.operr)
 
         last_instr = jit.promote(frame.last_instr)
         if last_instr != -1:
+            assert last_instr & 1 == 0
             frame.pushvalue(w_arg_or_err)
-        return r_uint(last_instr + 1)
+            return r_uint(last_instr + 2)
+        else:
+            return r_uint(0)
 
     def next_yield_from(self, frame, w_yf, w_inputvalue_or_err):
         """Fetch the next item of the current 'yield from', push it on
diff --git a/pypy/interpreter/interactive.py b/pypy/interpreter/interactive.py
--- a/pypy/interpreter/interactive.py
+++ b/pypy/interpreter/interactive.py
@@ -213,7 +213,7 @@
             ec.bytecode_only_trace = self._orig_bytecode_only_trace
 
     def _do_bytecode_only_trace(self, frame):
-        from pypy.tool.pydis import Bytecode, HAVE_ARGUMENT
+        from pypy.tool import opcode3, dis3
 
         if frame.hide():
             return
@@ -221,18 +221,12 @@
         self.unsettrace()
         next_instr = frame.last_instr
         opcode = ord(frame.pycode.co_code[next_instr])
+        oparg = ord(frame.pycode.co_code[next_instr+1])
 
-        oparg = 0
-        if opcode >= HAVE_ARGUMENT:
-            lo = ord(frame.pycode.co_code[next_instr+1])
-            hi = ord(frame.pycode.co_code[next_instr+2])
-            oparg = (hi * 256) | lo
-
-        class fake:
-            code = frame.pycode
-        bytecode = Bytecode(fake, next_instr, oparg, 0)
+        argrepr = reprargstring(self.space, frame.pycode, opcode, oparg)
+        oprepr = opcode3.opname[opcode] + argrepr.ljust(5)
         print '\t%-19s %s' % (str(frame.pycode.co_name) + ':',
-                              bytecode.repr_with_space(self.space))
+                              oprepr)
         self.settrace()
 
     def checktrace(self):
@@ -255,3 +249,26 @@
 
 class IncompleteInput(Exception):
     pass
+
+
+def reprargstring(space, pycode, opcode, oparg):
+    """ return a string representation of any arguments. (empty for no args)"""
+    from pypy.tool import opcode3
+    if oparg is None:
+        return ''
+    s = repr(oparg).rjust(5) + " "
+    if opcode in opcode3.hasconst:
+        r = space.text_w(space.repr(pycode.co_consts_w[oparg]))
+        s += '(' + r + ')'
+    elif opcode in opcode3.hasname:
+        s +=  '(' + pycode.co_names[oparg] + ')'
+    elif opcode in opcode3.hasjrel:
+        s +=  '(to ' + repr(self.index + oparg) + ')'
+    elif opcode in opcode3.haslocal:
+        s +=  '(' + pycode.co_varnames[oparg] + ')'
+    elif opcode in opcode3.hascompare:
+        s +=  '(' + opcode3.cmp_op[oparg] + ')'
+    elif opcode in opcode3.hasfree:
+        free = pycode.co_cellvars + pycode.co_freevars
+        s +=  '(' + free[oparg] + ')'
+    return s
diff --git a/pypy/interpreter/pycode.py b/pypy/interpreter/pycode.py
--- a/pypy/interpreter/pycode.py
+++ b/pypy/interpreter/pycode.py
@@ -39,7 +39,7 @@
 # time you make pyc files incompatible.  This value ends up in the frozen
 # importlib, via MAGIC_NUMBER in module/_frozen_importlib/__init__.
 
-pypy_incremental_magic = 128 # bump it by 16
+pypy_incremental_magic = 160 # bump it by 16
 assert pypy_incremental_magic % 16 == 0
 assert pypy_incremental_magic < 3000 # the magic number of Python 3. There are
                                      # no known magic numbers below this value
@@ -216,6 +216,7 @@
         """
         Hack to initialize the code object from a real (CPython) one.
         """
+        raise TypeError("assert reinterpretation for applevel tests is broken on PyPy3!")
         assert isinstance(code, types.CodeType)
         newconsts_w = [None] * len(code.co_consts)
         num = 0
@@ -301,11 +302,7 @@
                 w_co.remove_docstrings(space)
 
     def exec_host_bytecode(self, w_globals, w_locals):
-        if sys.version_info < (2, 7):
-            raise Exception("PyPy no longer supports Python 2.6 or lower")
-        frame = self.space.FrameClass(self.space, self, w_globals, None)
-        frame.setdictscope(w_locals)
-        return frame.run()
+        raise Exception("no longer supported after the switch to wordcode!")
 
     def dump(self):
         """NOT_RPYTHON: A dis.dis() dump of the code object."""
diff --git a/pypy/interpreter/pyframe.py b/pypy/interpreter/pyframe.py
--- a/pypy/interpreter/pyframe.py
+++ b/pypy/interpreter/pyframe.py
@@ -691,6 +691,7 @@
         endblock = [-1]     # current finally/except block stack
         addr = 0
         while addr < len(code):
+            assert addr & 1 == 0
             op = ord(code[addr])
             if op in (SETUP_LOOP, SETUP_EXCEPT, SETUP_FINALLY, SETUP_WITH,
                       SETUP_ASYNC_WITH):
@@ -713,10 +714,7 @@
             if addr == self.last_instr:
                 f_lasti_handler_addr = endblock[-1]
 
-            if op >= HAVE_ARGUMENT:
-                addr += 3
-            else:
-                addr += 1
+            addr += 2
 
         if len(blockstack) != 0 or len(endblock) != 1:
             raise oefmt(space.w_SystemError,
@@ -774,6 +772,7 @@
             block.cleanupstack(self)
 
         self.getorcreatedebug().f_lineno = new_lineno
+        assert new_lasti & 1 == 0
         self.last_instr = new_lasti
 
     def get_last_lineno(self):
diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py
--- a/pypy/interpreter/pyopcode.py
+++ b/pypy/interpreter/pyopcode.py
@@ -62,6 +62,7 @@
         co_code = pycode.co_code
         try:
             while True:
+                assert next_instr & 1 == 0
                 next_instr = self.handle_bytecode(co_code, next_instr, ec)
         except ExitFrame:
             return self.popvalue()
@@ -152,22 +153,17 @@
     @jit.unroll_safe
     def dispatch_bytecode(self, co_code, next_instr, ec):
         while True:
+            assert next_instr & 1 == 0
             self.last_instr = intmask(next_instr)
             if jit.we_are_jitted():
                 ec.bytecode_only_trace(self)
             else:
                 ec.bytecode_trace(self)
             next_instr = r_uint(self.last_instr)
+            assert next_instr & 1 == 0
             opcode = ord(co_code[next_instr])
-            next_instr += 1
-
-            if opcode >= HAVE_ARGUMENT:
-                lo = ord(co_code[next_instr])
-                hi = ord(co_code[next_instr+1])
-                next_instr += 2
-                oparg = (hi * 256) | lo
-            else:
-                oparg = 0
+            oparg = ord(co_code[next_instr + 1])
+            next_instr += 2
 
             # note: the structure of the code here is such that it makes
             # (after translation) a big "if/elif" chain, which is then
@@ -175,12 +171,11 @@
 
             while opcode == opcodedesc.EXTENDED_ARG.index:
                 opcode = ord(co_code[next_instr])
+                arg = ord(co_code[next_instr + 1])
                 if opcode < HAVE_ARGUMENT:
                     raise BytecodeCorruption
-                lo = ord(co_code[next_instr+1])
-                hi = ord(co_code[next_instr+2])
-                next_instr += 3
-                oparg = (oparg * 65536) | (hi * 256) | lo
+                next_instr += 2
+                oparg = (oparg * 256) | arg
 
             if opcode == opcodedesc.RETURN_VALUE.index:
                 w_returnvalue = self.popvalue()
@@ -252,6 +247,8 @@
                 self.BINARY_TRUE_DIVIDE(oparg, next_instr)
             elif opcode == opcodedesc.BINARY_XOR.index:
                 self.BINARY_XOR(oparg, next_instr)
+            elif opcode == opcodedesc.BUILD_CONST_KEY_MAP.index:
+                self.BUILD_CONST_KEY_MAP(oparg, next_instr)
             elif opcode == opcodedesc.BUILD_LIST.index:
                 self.BUILD_LIST(oparg, next_instr)
             elif opcode == opcodedesc.BUILD_LIST_FROM_ARG.index:
@@ -362,8 +359,6 @@
                 self.LOAD_NAME(oparg, next_instr)
             elif opcode == opcodedesc.LOOKUP_METHOD.index:
                 self.LOOKUP_METHOD(oparg, next_instr)
-            elif opcode == opcodedesc.MAKE_CLOSURE.index:
-                self.MAKE_CLOSURE(oparg, next_instr)
             elif opcode == opcodedesc.MAKE_FUNCTION.index:
                 self.MAKE_FUNCTION(oparg, next_instr)
             elif opcode == opcodedesc.MAP_ADD.index:
@@ -1357,47 +1352,42 @@
         self.call_function(oparg, w_varkw, has_vararg=True)
 
     @jit.unroll_safe
-    def _make_function(self, oparg, freevars=None):
+    def MAKE_FUNCTION(self, oparg, next_instr):
         space = self.space
         w_qualname = self.popvalue()
         qualname = self.space.unicode_w(w_qualname)
         w_codeobj = self.popvalue()
         codeobj = self.space.interp_w(PyCode, w_codeobj)
-        if freevars is not None:
-            # Pop freevars
-            self.popvalue()
-        posdefaults = oparg & 0xFF
-        kwdefaults = (oparg >> 8) & 0xFF
-        num_annotations = (oparg >> 16) & 0xFF
-        w_ann = None
-        if num_annotations:
-            names_w = space.fixedview(self.popvalue())
-            w_ann = space.newdict(strdict=True)
-            for i in range(len(names_w) - 1, -1, -1):
-                space.setitem(w_ann, names_w[i], self.popvalue())
-        kw_defs_w = None
-        if kwdefaults:
-            kw_defs_w = []
-            for i in range(kwdefaults):
-                w_defvalue = self.popvalue()
-                w_defname = self.popvalue()
-                kw_defs_w.append((w_defname, w_defvalue))
-        defaultarguments = self.popvalues(posdefaults)
+        assert 0 <= oparg <= 0x0F
+        if oparg & 0x08:
+            w_freevarstuple = self.popvalue()
+            # XXX this list copy is expensive, it's purely for the annotator
+            freevars = [self.space.interp_w(Cell, cell)
+                        for cell in self.space.fixedview(w_freevarstuple)]
+        else:
+            freevars = None
+        if oparg & 0x04:
+            w_ann = self.popvalue()
+        else:
+            w_ann = None
+        if oparg & 0x02:
+            w_kw_defs = self.popvalue()
+            # XXX
+            kw_defs_w = [space.unpackiterable(w_tup)
+                            for w_tup in space.fixedview(
+                                space.call_method(w_kw_defs, 'items'))]
+        else:
+            kw_defs_w = None
+        if oparg & 0x01:
+            defaultarguments = space.fixedview(self.popvalue())
+        else:
+            defaultarguments = []
+
         fn = function.Function(space, codeobj, self.get_w_globals(),
                                defaultarguments,
                                kw_defs_w, freevars, w_ann, qualname=qualname)
         self.pushvalue(fn)
 
-    def MAKE_FUNCTION(self, oparg, next_instr):
-        return self._make_function(oparg)
-
-    @jit.unroll_safe
-    def MAKE_CLOSURE(self, oparg, next_instr):
-        w_freevarstuple = self.peekvalue(2)
-        freevars = [self.space.interp_w(Cell, cell)
-                    for cell in self.space.fixedview(w_freevarstuple)]
-        self._make_function(oparg, freevars)
-
     def BUILD_SLICE(self, numargs, next_instr):
         if numargs == 3:
             w_step = self.popvalue()
@@ -1447,7 +1437,18 @@
             w_value = self.peekvalue(2 * i)
             w_key = self.peekvalue(2 * i + 1)
             self.space.setitem(w_dict, w_key, w_value)
-        self.popvalues(2 * itemcount)
+        self.dropvalues(2 * itemcount)
+        self.pushvalue(w_dict)
+
+    @jit.unroll_safe
+    def BUILD_CONST_KEY_MAP(self, itemcount, next_instr):
+        keys_w = self.space.fixedview(self.popvalue())
+        w_dict = self.space.newdict()
+        for i in range(itemcount):
+            w_value = self.peekvalue(itemcount - 1 - i)
+            w_key = keys_w[i]
+            self.space.setitem(w_dict, w_key, w_value)
+        self.dropvalues(itemcount)
         self.pushvalue(w_dict)
 
     @jit.unroll_safe
@@ -1456,7 +1457,7 @@
         for i in range(itemcount-1, -1, -1):
             w_item = self.peekvalue(i)
             self.space.call_method(w_set, 'add', w_item)
-        self.popvalues(itemcount)
+        self.dropvalues(itemcount)
         self.pushvalue(w_set)
 
     @jit.unroll_safe
diff --git a/pypy/interpreter/test/test_annotations.py b/pypy/interpreter/test/test_annotations.py
--- a/pypy/interpreter/test/test_annotations.py
+++ b/pypy/interpreter/test/test_annotations.py
@@ -132,3 +132,11 @@
             ''')
         """)
 
+    def test_lineno(self):
+        s = """
+
+a: int
+        """
+        c = compile(s, "f", "exec")
+        assert c.co_firstlineno == 3
+
diff --git a/pypy/interpreter/test/test_function.py b/pypy/interpreter/test/test_function.py
--- a/pypy/interpreter/test/test_function.py
+++ b/pypy/interpreter/test/test_function.py
@@ -643,11 +643,20 @@
 
 
 class TestMethod:
-    def setup_method(self, method):
-        def c(self, bar):
-            return bar
-        code = PyCode._from_code(self.space, c.__code__)
-        self.fn = Function(self.space, code, self.space.newdict())
+    @classmethod
+    def compile(cls, src):
+        assert src.strip().startswith("def ")
+        compiler = cls.space.createcompiler()
+        code = compiler.compile(src, '<hello>', 'exec', 0).co_consts_w[0]
+        return Function(cls.space, code, cls.space.newdict())
+
+    def setup_class(cls):
+        src = """
+def c(self, bar):
+    return bar
+        """
+        cls.fn = cls.compile(src)
+
 
     def test_get(self):
         space = self.space
@@ -672,9 +681,7 @@
     def test_method_get(self):
         space = self.space
         # Create some function for this test only
-        def m(self): return self
-        func = Function(space, PyCode._from_code(self.space, m.__code__),
-                        space.newdict())
+        func = self.compile("def m(self): return self")
         # Some shorthands
         obj1 = space.wrap(23)
         obj2 = space.wrap(42)
@@ -696,6 +703,11 @@
         assert meth3 is func
 
 class TestShortcuts(object):
+    def compile(self, src):
+        assert src.strip().startswith("def ")
+        compiler = self.space.createcompiler()
+        code = compiler.compile(src, '<hello>', 'exec', 0).co_consts_w[0]
+        return Function(self.space, code, self.space.newdict())
 
     def test_call_function(self):
         space = self.space
@@ -703,14 +715,15 @@
         d = {}
         for i in range(10):
             args = "(" + ''.join(["a%d," % a for a in range(i)]) + ")"
-            exec """
+            src = """
 def f%s:
     return %s
-""" % (args, args) in d
+""" % (args, args)
+            exec src in d
             f = d['f']
             res = f(*range(i))
-            code = PyCode._from_code(self.space, f.__code__)
-            fn = Function(self.space, code, self.space.newdict())
+            fn = self.compile(src)
+            code = fn.code
 
             assert fn.code.fast_natural_arity == i|PyCode.FLATPYCALL
             if i < 5:
@@ -729,18 +742,18 @@
     def test_flatcall(self):
         space = self.space
 
-        def f(a):
-            return a
-        code = PyCode._from_code(self.space, f.__code__)
-        fn = Function(self.space, code, self.space.newdict())
+        src = """
+def f(a):
+    return a"""
+        fn = self.compile(src)
 
         assert fn.code.fast_natural_arity == 1|PyCode.FLATPYCALL
 
         def bomb(*args):
             assert False, "shortcutting should have avoided this"
 
-        code.funcrun = bomb
-        code.funcrun_obj = bomb
+        fn.code.funcrun = bomb
+        fn.code.funcrun_obj = bomb
 
         w_3 = space.newint(3)
         w_res = space.call_function(fn, w_3)
@@ -756,18 +769,19 @@
     def test_flatcall_method(self):
         space = self.space
 
-        def f(self, a):
-            return a
-        code = PyCode._from_code(self.space, f.__code__)
-        fn = Function(self.space, code, self.space.newdict())
+        src = """
+def f(self, a):
+    return a
+"""
+        fn = self.compile(src)
 
         assert fn.code.fast_natural_arity == 2|PyCode.FLATPYCALL
 
         def bomb(*args):
             assert False, "shortcutting should have avoided this"
 
-        code.funcrun = bomb
-        code.funcrun_obj = bomb
+        fn.code.funcrun = bomb
+        fn.code.funcrun_obj = bomb
 
         w_3 = space.newint(3)
         w_res = space.appexec([fn, w_3], """(f, x):
@@ -784,9 +798,11 @@
     def test_flatcall_default_arg(self):
         space = self.space
 
-        def f(a, b):
-            return a+b
-        code = PyCode._from_code(self.space, f.__code__)
+        src = """
+def f(a, b):
+    return a+b
+"""
+        code = self.compile(src).code
         fn = Function(self.space, code, self.space.newdict(),
                       defs_w=[space.newint(1)])
 
@@ -813,9 +829,11 @@
     def test_flatcall_default_arg_method(self):
         space = self.space
 
-        def f(self, a, b):
-            return a+b
-        code = PyCode._from_code(self.space, f.__code__)
+        src = """
+def f(self, a, b):
+    return a+b
+        """
+        code = self.compile(src).code
         fn = Function(self.space, code, self.space.newdict(),
                       defs_w=[space.newint(1)])
 
diff --git a/pypy/interpreter/test/test_pycode.py b/pypy/interpreter/test/test_pycode.py
--- a/pypy/interpreter/test/test_pycode.py
+++ b/pypy/interpreter/test/test_pycode.py
@@ -14,6 +14,6 @@
     finally:
         sys.stdout = stdout
     print '>>>\n' + output + '\n<<<'
-    assert ' 1 (7)' in output
+    assert ' 0 (7)' in output
     assert ' 4 (None)' in output
-    assert ' 19 RETURN_VALUE ' in output
+    assert ' 16 RETURN_VALUE' in output
diff --git a/pypy/interpreter/test/test_zpy.py b/pypy/interpreter/test/test_zpy.py
--- a/pypy/interpreter/test/test_zpy.py
+++ b/pypy/interpreter/test/test_zpy.py
@@ -122,6 +122,3 @@
     # '5\n' --- this line sent to stderr
     assert ('\t<module>:           LOAD_NAME    0 (x)\n'
             '\t<module>:           PRINT_EXPR    0 \n') in output
-    assert ('\t<module>:           LOAD_CONST    0 (None)\n'
-            '\t<module>:           RETURN_VALUE    0 \n'
-            '>>>> ') in output
diff --git a/pypy/module/_ast/test/test_ast.py b/pypy/module/_ast/test/test_ast.py
--- a/pypy/module/_ast/test/test_ast.py
+++ b/pypy/module/_ast/test/test_ast.py
@@ -461,7 +461,7 @@
 
     def test_bug_null_in_objspace_type(self):
         import ast
-        code = ast.Expression(lineno=1, col_offset=1, body=ast.ListComp(lineno=1, col_offset=1, elt=ast.Call(lineno=1, col_offset=1, func=ast.Name(lineno=1, col_offset=1, id='str', ctx=ast.Load(lineno=1, col_offset=1)), args=[ast.Name(lineno=1, col_offset=1, id='x', ctx=ast.Load(lineno=1, col_offset=1))], keywords=[]), generators=[ast.comprehension(lineno=1, col_offset=1, target=ast.Name(lineno=1, col_offset=1, id='x', ctx=ast.Store(lineno=1, col_offset=1)), iter=ast.List(lineno=1, col_offset=1, elts=[ast.Num(lineno=1, col_offset=1, n=23)], ctx=ast.Load(lineno=1, col_offset=1, )), ifs=[])]))
+        code = ast.Expression(lineno=1, col_offset=1, body=ast.ListComp(lineno=1, col_offset=1, elt=ast.Call(lineno=1, col_offset=1, func=ast.Name(lineno=1, col_offset=1, id='str', ctx=ast.Load(lineno=1, col_offset=1)), args=[ast.Name(lineno=1, col_offset=1, id='x', ctx=ast.Load(lineno=1, col_offset=1))], keywords=[]), generators=[ast.comprehension(lineno=1, col_offset=1, target=ast.Name(lineno=1, col_offset=1, id='x', ctx=ast.Store(lineno=1, col_offset=1)), iter=ast.List(lineno=1, col_offset=1, elts=[ast.Num(lineno=1, col_offset=1, n=23)], ctx=ast.Load(lineno=1, col_offset=1, )), ifs=[], is_async=False)]))
         compile(code, '<template>', 'eval')
 
     def test_empty_yield_from(self):
diff --git a/pypy/module/imp/test/test_import.py b/pypy/module/imp/test/test_import.py
--- a/pypy/module/imp/test/test_import.py
+++ b/pypy/module/imp/test/test_import.py
@@ -788,7 +788,7 @@
     f.write(_getlong(mtime))
     if co:
         # marshal the code object with the PyPy marshal impl
-        pyco = PyCode._from_code(space, co)
+        pyco = space.createcompiler().compile(co, '?', 'exec', 0)
         w_marshal = space.getbuiltinmodule('marshal')
         w_marshaled_code = space.call_method(w_marshal, 'dumps', pyco)
         marshaled_code = space.bytes_w(w_marshaled_code)
@@ -809,7 +809,7 @@
     def test_read_compiled_module(self):
         space = self.space
         mtime = 12345
-        co = compile('x = 42', '?', 'exec')
+        co = 'x = 42'
         cpathname = _testfile(space, importing.get_pyc_magic(space), mtime, co)
         stream = streamio.open_file_as_stream(cpathname, "rb")
         try:
@@ -829,7 +829,7 @@
     def test_load_compiled_module(self):
         space = self.space
         mtime = 12345
-        co = compile('x = 42', '?', 'exec')
+        co = 'x = 42'
         cpathname = _testfile(space, importing.get_pyc_magic(space), mtime, co)
         w_modulename = space.wrap('somemodule')
         stream = streamio.open_file_as_stream(cpathname, "rb")
@@ -854,7 +854,7 @@
     def test_load_compiled_module_nopathname(self):
         space = self.space
         mtime = 12345
-        co = compile('x = 42', '?', 'exec')
+        co = 'x = 42'
         cpathname = _testfile(space, importing.get_pyc_magic(space), mtime, co)
         w_modulename = space.wrap('somemodule')
         stream = streamio.open_file_as_stream(cpathname, "rb")
@@ -931,7 +931,7 @@
                     continue
                 pathname = "whatever"
                 mtime = 12345
-                co = compile('x = 42', '?', 'exec')
+                co = 'x = 42'
                 cpathname = _testfile(space1, importing.get_pyc_magic(space1),
                                       mtime, co)
                 w_modulename = space2.wrap('somemodule')
diff --git a/pypy/tool/dis3.py b/pypy/tool/dis3.py
--- a/pypy/tool/dis3.py
+++ b/pypy/tool/dis3.py
@@ -1,20 +1,40 @@
-"""Disassembler of Python byte code into mnemonics.
-Python 3 dis.py partly backported to Python 2"""
+"""Disassembler of Python byte code into mnemonics."""
+
+from __future__ import print_function
 
 import sys
 import types
+import collections
+import io
 
 from opcode3 import *
 from opcode3 import __all__ as _opcodes_all
 
-__all__ = ["dis", "disassemble", "distb", "disco",
-           "findlinestarts", "findlabels"] + _opcodes_all
+__all__ = ["code_info", "dis", "disassemble", "distb", "disco",
+           "findlinestarts", "findlabels", "show_code",
+           "get_instructions", "Instruction", "Bytecode"] + _opcodes_all
 del _opcodes_all
 
-_have_code = (types.MethodType, types.FunctionType, types.CodeType, type)
+_have_code = (types.MethodType, types.FunctionType, types.CodeType,
+              classmethod, staticmethod, type)
+
+FORMAT_VALUE = opmap['FORMAT_VALUE']
+
+def _try_compile(source, name):
+    """Attempts to compile the given source, first as an expression and
+       then as a statement if the first approach fails.
+
+       Utility function to accept strings in functions that otherwise
+       expect code objects
+    """
+    try:
+        c = compile(source, name, 'eval')
+    except SyntaxError:
+        c = compile(source, name, 'exec')
+    return c
 
 def dis(x=None):
-    """Disassemble classes, methods, functions, or code.
+    """Disassemble classes, methods, functions, generators, or code.
 
     With no argument, disassemble the last traceback.
 
@@ -22,30 +42,31 @@
     if x is None:
         distb()
         return
-    if isinstance(x, types.InstanceType):
-        x = x.__class__
-    if hasattr(x, 'im_func'):
-        x = x.im_func
-    if hasattr(x, 'func_code'):
-        x = x.func_code
-    if hasattr(x, 'co_code'): # PyCode needs co_code before __dict__
+    if hasattr(x, '__func__'):  # Method
+        x = x.__func__
+    if hasattr(x, '__code__'):  # Function
+        x = x.__code__
+    if hasattr(x, 'gi_code'):  # Generator
+        x = x.gi_code
+    if hasattr(x, 'co_code'): # Code object
         disassemble(x)
-    elif hasattr(x, '__dict__'):
-        items = x.__dict__.items()
-        items.sort()
+    elif hasattr(x, '__dict__'):  # Class or module
+        items = sorted(x.__dict__.items())
         for name, x1 in items:
             if isinstance(x1, _have_code):
-                print "Disassembly of %s:" % name
+                print("Disassembly of %s:" % name)
                 try:
                     dis(x1)
                 except TypeError as msg:
-                    print "Sorry:", msg
-                print
-    elif isinstance(x, str):
-        disassemble_string(x)
+                    print("Sorry:", msg)
+                print()
+    elif isinstance(x, (bytes, bytearray)): # Raw bytecode
+        _disassemble_bytes(x)
+    elif isinstance(x, str):    # Source code
+        _disassemble_str(x)
     else:
-        raise TypeError("don't know how to disassemble %s objects" % \
-              type(x).__name__)
+        raise TypeError("don't know how to disassemble %s objects" %
+                        type(x).__name__)
 
 def distb(tb=None):
     """Disassemble a traceback (default: last traceback)."""
@@ -57,103 +78,290 @@
         while tb.tb_next: tb = tb.tb_next
     disassemble(tb.tb_frame.f_code, tb.tb_lasti)
 
+# The inspect module interrogates this dictionary to build its
+# list of CO_* constants. It is also used by pretty_flags to
+# turn the co_flags field into a human readable list.
+COMPILER_FLAG_NAMES = {
+     1: "OPTIMIZED",
+     2: "NEWLOCALS",
+     4: "VARARGS",
+     8: "VARKEYWORDS",
+    16: "NESTED",
+    32: "GENERATOR",
+    64: "NOFREE",
+   128: "COROUTINE",
+   256: "ITERABLE_COROUTINE",
+   512: "ASYNC_GENERATOR",
+}
+
+def pretty_flags(flags):
+    """Return pretty representation of code flags."""
+    names = []
+    for i in range(32):
+        flag = 1<<i
+        if flags & flag:
+            names.append(COMPILER_FLAG_NAMES.get(flag, hex(flag)))
+            flags ^= flag
+            if not flags:
+                break
+    else:
+        names.append(hex(flags))
+    return ", ".join(names)
+
+def _get_code_object(x):
+    """Helper to handle methods, functions, generators, strings and raw code objects"""
+    if hasattr(x, '__func__'): # Method
+        x = x.__func__
+    if hasattr(x, '__code__'): # Function
+        x = x.__code__
+    if hasattr(x, 'gi_code'):  # Generator
+        x = x.gi_code
+    if isinstance(x, str):     # Source code
+        x = _try_compile(x, "<disassembly>")
+    if hasattr(x, 'co_code'):  # Code object
+        return x
+    raise TypeError("don't know how to disassemble %s objects" %
+                    type(x).__name__)
+
+def code_info(x):
+    """Formatted details of methods, functions, or code."""
+    return _format_code_info(_get_code_object(x))
+
+def _format_code_info(co):
+    lines = []
+    lines.append("Name:              %s" % co.co_name)
+    lines.append("Filename:          %s" % co.co_filename)
+    lines.append("Argument count:    %s" % co.co_argcount)
+    lines.append("Kw-only arguments: %s" % co.co_kwonlyargcount)
+    lines.append("Number of locals:  %s" % co.co_nlocals)
+    lines.append("Stack size:        %s" % co.co_stacksize)
+    lines.append("Flags:             %s" % pretty_flags(co.co_flags))
+    if co.co_consts:
+        lines.append("Constants:")
+        for i_c in enumerate(co.co_consts):
+            lines.append("%4d: %r" % i_c)
+    if co.co_names:
+        lines.append("Names:")
+        for i_n in enumerate(co.co_names):
+            lines.append("%4d: %s" % i_n)
+    if co.co_varnames:
+        lines.append("Variable names:")
+        for i_n in enumerate(co.co_varnames):
+            lines.append("%4d: %s" % i_n)
+    if co.co_freevars:
+        lines.append("Free variables:")
+        for i_n in enumerate(co.co_freevars):
+            lines.append("%4d: %s" % i_n)
+    if co.co_cellvars:
+        lines.append("Cell variables:")
+        for i_n in enumerate(co.co_cellvars):
+            lines.append("%4d: %s" % i_n)
+    return "\n".join(lines)
+
+def show_code(co, file=None):
+    """Print details of methods, functions, or code to *file*.
+
+    If *file* is not provided, the output is printed on stdout.
+    """
+    print(code_info(co))
+
+_Instruction = collections.namedtuple("_Instruction",
+     "opname opcode arg argval argrepr offset starts_line is_jump_target")
+
+class Instruction(_Instruction):
+    """Details for a bytecode operation
+
+       Defined fields:
+         opname - human readable name for operation
+         opcode - numeric code for operation
+         arg - numeric argument to operation (if any), otherwise None
+         argval - resolved arg value (if known), otherwise same as arg
+         argrepr - human readable description of operation argument
+         offset - start index of operation within bytecode sequence
+         starts_line - line started by this opcode (if any), otherwise None
+         is_jump_target - True if other code jumps to here, otherwise False
+    """
+
+    def _disassemble(self, lineno_width=3, mark_as_current=False):
+        """Format instruction details for inclusion in disassembly output
+
+        *lineno_width* sets the width of the line number field (0 omits it)
+        *mark_as_current* inserts a '-->' marker arrow as part of the line
+        """
+        fields = []
+        # Column: Source code line number
+        if lineno_width:
+            if self.starts_line is not None:
+                lineno_fmt = "%%%dd" % lineno_width
+                fields.append(lineno_fmt % self.starts_line)
+            else:
+                fields.append(' ' * lineno_width)
+        # Column: Current instruction indicator
+        if mark_as_current:
+            fields.append('-->')
+        else:
+            fields.append('   ')
+        # Column: Jump target marker
+        if self.is_jump_target:
+            fields.append('>>')
+        else:
+            fields.append('  ')
+        # Column: Instruction offset from start of code sequence
+        fields.append(repr(self.offset).rjust(4))
+        # Column: Opcode name
+        fields.append(self.opname.ljust(20))
+        # Column: Opcode argument
+        if self.arg is not None:
+            fields.append(repr(self.arg).rjust(5))
+            # Column: Opcode argument details
+            if self.argrepr:
+                fields.append('(' + self.argrepr + ')')
+        return ' '.join(fields).rstrip()
+
+
+def get_instructions(x, first_line=None):
+    """Iterator for the opcodes in methods, functions or code
+
+    Generates a series of Instruction named tuples giving the details of
+    each operations in the supplied code.
+
+    If *first_line* is not None, it indicates the line number that should
+    be reported for the first source line in the disassembled code.
+    Otherwise, the source line information (if any) is taken directly from
+    the disassembled code object.
+    """
+    co = _get_code_object(x)
+    cell_names = co.co_cellvars + co.co_freevars
+    linestarts = dict(findlinestarts(co))
+    if first_line is not None:
+        line_offset = first_line - co.co_firstlineno
+    else:
+        line_offset = 0
+    return _get_instructions_bytes(co.co_code, co.co_varnames, co.co_names,
+                                   co.co_consts, cell_names, linestarts,
+                                   line_offset)
+
+def _get_const_info(const_index, const_list):
+    """Helper to get optional details about const references
+
+       Returns the dereferenced constant and its repr if the constant
+       list is defined.
+       Otherwise returns the constant index and its repr().
+    """
+    argval = const_index
+    if const_list is not None:
+        argval = const_list[const_index]
+    return argval, repr(argval)
+
+def _get_name_info(name_index, name_list):
+    """Helper to get optional details about named references
+
+       Returns the dereferenced name as both value and repr if the name
+       list is defined.
+       Otherwise returns the name index and its repr().
+    """
+    argval = name_index
+    if name_list is not None:
+        argval = name_list[name_index]
+        argrepr = argval
+    else:
+        argrepr = repr(argval)
+    return argval, argrepr
+
+
+def _get_instructions_bytes(code, varnames=None, names=None, constants=None,
+                      cells=None, linestarts=None, line_offset=0):
+    """Iterate over the instructions in a bytecode string.
+
+    Generates a sequence of Instruction namedtuples giving the details of each
+    opcode.  Additional information about the code's runtime environment
+    (e.g. variable names, constants) can be specified using optional
+    arguments.
+
+    """
+    labels = findlabels(code)
+    starts_line = None
+    for offset, op, arg in _unpack_opargs(code):
+        if linestarts is not None:
+            starts_line = linestarts.get(offset, None)
+            if starts_line is not None:
+                starts_line += line_offset
+        is_jump_target = offset in labels
+        argval = None
+        argrepr = ''
+        if arg is not None:
+            #  Set argval to the dereferenced value of the argument when
+            #  available, and argrepr to the string representation of argval.
+            #    _disassemble_bytes needs the string repr of the
+            #    raw name index for LOAD_GLOBAL, LOAD_CONST, etc.
+            argval = arg
+            if op in hasconst:
+                argval, argrepr = _get_const_info(arg, constants)
+            elif op in hasname:
+                argval, argrepr = _get_name_info(arg, names)
+            elif op in hasjrel:
+                argval = offset + 2 + arg
+                argrepr = "to " + repr(argval)
+            elif op in haslocal:
+                argval, argrepr = _get_name_info(arg, varnames)
+            elif op in hascompare:
+                argval = cmp_op[arg]
+                argrepr = argval
+            elif op in hasfree:
+                argval, argrepr = _get_name_info(arg, cells)
+            elif op == FORMAT_VALUE:
+                argval = ((None, str, repr, ascii)[arg & 0x3], bool(arg & 0x4))
+                argrepr = ('', 'str', 'repr', 'ascii')[arg & 0x3]
+                if argval[1]:
+                    if argrepr:
+                        argrepr += ', '
+                    argrepr += 'with format'
+        yield Instruction(opname[op], op,
+                          arg, argval, argrepr,
+                          offset, starts_line, is_jump_target)
+
 def disassemble(co, lasti=-1):
     """Disassemble a code object."""
-    code = co.co_code
-    labels = findlabels(code)
+    cell_names = co.co_cellvars + co.co_freevars
     linestarts = dict(findlinestarts(co))
-    n = len(code)
-    i = 0
-    extended_arg = 0
-    free = None
-    while i < n:
-        c = code[i]
-        op = ord(c)
-        if i in linestarts:
-            if i > 0:
-                print
-            print "%3d" % linestarts[i],
-        else:
-            print '   ',
+    _disassemble_bytes(co.co_code, lasti, co.co_varnames, co.co_names,
+                       co.co_consts, cell_names, linestarts)
 
-        if i == lasti: print '-->',
-        else: print '   ',
-        if i in labels: print '>>',
-        else: print '  ',
-        print repr(i).rjust(4),
-        print opname[op].ljust(20),
-        i = i+1
-        if op >= HAVE_ARGUMENT:
-            oparg = ord(code[i]) + ord(code[i+1])*256 + extended_arg
-            extended_arg = 0
-            i = i+2
-            if op == EXTENDED_ARG:
-                extended_arg = oparg*65536L
-            print repr(oparg).rjust(5),
-            if op in hasconst:
-                print '(' + repr(co.co_consts[oparg]) + ')',
-            elif op in hasname:
-                print '(' + co.co_names[oparg] + ')',
-            elif op in hasjrel:
-                print '(to ' + repr(i + oparg) + ')',
-            elif op in haslocal:
-                print '(' + co.co_varnames[oparg] + ')',
-            elif op in hascompare:
-                print '(' + cmp_op[oparg] + ')',
-            elif op in hasfree:
-                if free is None:
-                    free = co.co_cellvars + co.co_freevars
-                print '(' + free[oparg] + ')',
-            elif op in hasnargs:
-                print '(%d positional, %d keyword pair)' % \
-                      (ord(code[i-2]), ord(code[i-1])),
-        print
+def _disassemble_bytes(code, lasti=-1, varnames=None, names=None,
+                       constants=None, cells=None, linestarts=None,
+                       line_offset=0):
+    # Omit the line number column entirely if we have no line number info
+    show_lineno = linestarts is not None
+    # TODO?: Adjust width upwards if max(linestarts.values()) >= 1000?
+    lineno_width = 3 if show_lineno else 0
+    for instr in _get_instructions_bytes(code, varnames, names,
+                                         constants, cells, linestarts,
+                                         line_offset=line_offset):
+        new_source_line = (show_lineno and
+                           instr.starts_line is not None and
+                           instr.offset > 0)
+        if new_source_line:
+            print()
+        is_current_instr = instr.offset == lasti
+        print(instr._disassemble(lineno_width, is_current_instr))
 
-def disassemble_string(code, lasti=-1, varnames=None, names=None,
-                       constants=None):
-    labels = findlabels(code)
-    n = len(code)
-    i = 0
-    while i < n:
-        c = code[i]
-        op = ord(c)
-        if i == lasti: print '-->',
-        else: print '   ',
-        if i in labels: print '>>',
-        else: print '  ',
-        print repr(i).rjust(4),
-        print opname[op].ljust(15),
-        i = i+1
-        if op >= HAVE_ARGUMENT:
-            oparg = ord(code[i]) + ord(code[i+1])*256
-            i = i+2
-            print repr(oparg).rjust(5),
-            if op in hasconst:
-                if constants:
-                    print '(' + repr(constants[oparg]) + ')',
-                else:
-                    print '(%d)'%oparg,
-            elif op in hasname:
-                if names is not None:
-                    print '(' + names[oparg] + ')',
-                else:
-                    print '(%d)'%oparg,
-            elif op in hasjrel:
-                print '(to ' + repr(i + oparg) + ')',
-            elif op in haslocal:
-                if varnames:
-                    print '(' + varnames[oparg] + ')',
-                else:
-                    print '(%d)' % oparg,
-            elif op in hascompare:
-                print '(' + cmp_op[oparg] + ')',
-            elif op in hasnargs:
-                print '(%d positional, %d keyword pair)' % \
-                      (ord(code[i-2]), ord(code[i-1])),
-        print
+def _disassemble_str(source):
+    """Compile the source string, then disassemble the code object."""
+    disassemble(_try_compile(source, '<dis>'))
 
 disco = disassemble                     # XXX For backwards compatibility
 
+def _unpack_opargs(code):
+    extended_arg = 0
+    for i in range(0, len(code), 2):
+        op = ord(code[i])
+        if op >= HAVE_ARGUMENT:
+            arg = ord(code[i+1]) | extended_arg
+            extended_arg = (arg << 8) if op == EXTENDED_ARG else 0
+        else:
+            arg = None
+        yield (i, op, arg)
+
 def findlabels(code):
     """Detect all offsets in a byte code which are jump targets.
 
@@ -161,23 +369,16 @@
 
     """
     labels = []
-    n = len(code)
-    i = 0
-    while i < n:
-        c = code[i]
-        op = ord(c)
-        i = i+1
-        if op >= HAVE_ARGUMENT:
-            oparg = ord(code[i]) + ord(code[i+1])*256
-            i = i+2
-            label = -1
+    for offset, op, arg in _unpack_opargs(code):
+        if arg is not None:
             if op in hasjrel:
-                label = i+oparg
+                label = offset + 2 + arg
             elif op in hasjabs:
-                label = oparg
-            if label >= 0:
-                if label not in labels:
-                    labels.append(label)
+                label = arg
+            else:
+                continue
+            if label not in labels:
+                labels.append(label)
     return labels
 
 def findlinestarts(code):
@@ -186,13 +387,15 @@
     Generate pairs (offset, lineno) as described in Python/compile.c.
 
     """
-    byte_increments = [ord(c) for c in code.co_lnotab[0::2]]
-    line_increments = [ord(c) for c in code.co_lnotab[1::2]]
+    byte_increments = code.co_lnotab[0::2]
+    line_increments = code.co_lnotab[1::2]
 
     lastlineno = None
     lineno = code.co_firstlineno
     addr = 0
     for byte_incr, line_incr in zip(byte_increments, line_increments):
+        byte_incr = ord(byte_incr)
+        line_incr = ord(line_incr)
         if byte_incr:
             if lineno != lastlineno:
                 yield (addr, lineno)
@@ -205,27 +408,77 @@
     if lineno != lastlineno:
         yield (addr, lineno)
 
+class Bytecode:
+    """The bytecode operations of a piece of code
+
+    Instantiate this with a function, method, string of code, or a code object
+    (as returned by compile()).
+
+    Iterating over this yields the bytecode operations as Instruction instances.
+    """
+    def __init__(self, x, first_line=None, current_offset=None):
+        self.codeobj = co = _get_code_object(x)
+        if first_line is None:
+            self.first_line = co.co_firstlineno
+            self._line_offset = 0
+        else:
+            self.first_line = first_line
+            self._line_offset = first_line - co.co_firstlineno
+        self._cell_names = co.co_cellvars + co.co_freevars
+        self._linestarts = dict(findlinestarts(co))
+        self._original_object = x
+        self.current_offset = current_offset
+
+    def __iter__(self):
+        co = self.codeobj
+        return _get_instructions_bytes(co.co_code, co.co_varnames, co.co_names,
+                                       co.co_consts, self._cell_names,
+                                       self._linestarts,
+                                       line_offset=self._line_offset)
+
+    def __repr__(self):
+        return "{}({!r})".format(self.__class__.__name__,
+                                 self._original_object)
+
+    @classmethod
+    def from_traceback(cls, tb):
+        """ Construct a Bytecode from the given traceback """
+        while tb.tb_next:
+            tb = tb.tb_next
+        return cls(tb.tb_frame.f_code, current_offset=tb.tb_lasti)
+
+    def info(self):
+        """Return formatted information about the code object."""
+        return _format_code_info(self.codeobj)
+
+    def dis(self):
+        """Return a formatted view of the bytecode operations."""
+        co = self.codeobj
+        if self.current_offset is not None:
+            offset = self.current_offset
+        else:
+            offset = -1
+        with io.StringIO() as output:
+            _disassemble_bytes(co.co_code, varnames=co.co_varnames,
+                               names=co.co_names, constants=co.co_consts,
+                               cells=self._cell_names,
+                               linestarts=self._linestarts,
+                               line_offset=self._line_offset,
+                               file=output,
+                               lasti=offset)
+            return output.getvalue()
+
+
 def _test():
     """Simple test program to disassemble a file."""
-    if sys.argv[1:]:
-        if sys.argv[2:]:
-            sys.stderr.write("usage: python dis.py [-|file]\n")
-            sys.exit(2)
-        fn = sys.argv[1]
-        if not fn or fn == "-":
-            fn = None
-    else:
-        fn = None
-    if fn is None:
-        f = sys.stdin
-    else:
-        f = open(fn)
-    source = f.read()
-    if fn is not None:
-        f.close()
-    else:
-        fn = "<stdin>"
-    code = compile(source, fn, "exec")
+    import argparse
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument('infile', type=argparse.FileType(), nargs='?', default='-')
+    args = parser.parse_args()
+    with args.infile as infile:
+        source = infile.read()
+    code = compile(source, args.infile.name, "exec")
     dis(code)
 
 if __name__ == "__main__":
diff --git a/pypy/tool/opcode3.py b/pypy/tool/opcode3.py
--- a/pypy/tool/opcode3.py
+++ b/pypy/tool/opcode3.py
@@ -2,10 +2,8 @@
 """
 opcode module - potentially shared between dis and other modules which
 operate on bytecodes (e.g. peephole optimizers).
-"Backported" from Python 3 to Python 2 land - an excact copy of lib-python/3/opcode.py
 """
 
-
 __all__ = ["cmp_op", "hasconst", "hasname", "hasjrel", "hasjabs",
            "haslocal", "hascompare", "hasfree", "opname", "opmap",
            "HAVE_ARGUMENT", "EXTENDED_ARG", "hasnargs"]
@@ -33,10 +31,12 @@
 haslocal = []
 hascompare = []
 hasfree = []
-hasnargs = [] # unused
+hasnargs = []
 
 opmap = {}
-opname = ['<%r>' % (op,) for op in range(256)]
+opname = [''] * 256
+for op in range(256): opname[op] = '<%r>' % (op,)
+del op
 
 def def_op(name, op):
     opname[op] = name
@@ -174,9 +174,11 @@
 name_op('STORE_ANNOTATION', 127) # Index in name list
 
 def_op('RAISE_VARARGS', 130)    # Number of raise arguments (1, 2, or 3)
-def_op('CALL_FUNCTION', 131)    # #args
-def_op('MAKE_FUNCTION', 132)    # Flags
+def_op('CALL_FUNCTION', 131)    # #args + (#kwargs << 8)
+hasnargs.append(131)
+def_op('MAKE_FUNCTION', 132)    # Number of args with default values
 def_op('BUILD_SLICE', 133)      # Number of items
+def_op('MAKE_CLOSURE', 134)
 def_op('LOAD_CLOSURE', 135)
 hasfree.append(135)
 def_op('LOAD_DEREF', 136)
@@ -186,8 +188,12 @@
 def_op('DELETE_DEREF', 138)
 hasfree.append(138)
 
-def_op('CALL_FUNCTION_KW', 141)  # #args + #kwargs
-def_op('CALL_FUNCTION_EX', 142)  # Flags
+def_op('CALL_FUNCTION_VAR', 140)     # #args + (#kwargs << 8)
+hasnargs.append(140)
+def_op('CALL_FUNCTION_KW', 141)      # #args + (#kwargs << 8)
+hasnargs.append(141)
+def_op('CALL_FUNCTION_VAR_KW', 142)  # #args + (#kwargs << 8)
+hasnargs.append(142)
 
 jrel_op('SETUP_WITH', 143)
 
@@ -198,6 +204,8 @@
 def_op('LOAD_CLASSDEREF', 148)
 hasfree.append(148)
 
+jrel_op('SETUP_ASYNC_WITH', 154)
+
 def_op('EXTENDED_ARG', 144)
 EXTENDED_ARG = 144
 
@@ -207,12 +215,9 @@
 def_op('BUILD_TUPLE_UNPACK', 152)
 def_op('BUILD_SET_UNPACK', 153)
 
-jrel_op('SETUP_ASYNC_WITH', 154)
-
-def_op('FORMAT_VALUE', 155)
-def_op('BUILD_CONST_KEY_MAP', 156)
-def_op('BUILD_STRING', 157)
-def_op('BUILD_TUPLE_UNPACK_WITH_CALL', 158)
+def_op('FORMAT_VALUE', 155)   # in CPython 3.6, but available in PyPy from 3.5
+def_op("BUILD_CONST_KEY_MAP", 156)
+def_op('BUILD_STRING', 157)   # in CPython 3.6, but available in PyPy from 3.5
 
 # pypy modification, experimental bytecode
 def_op('LOOKUP_METHOD', 201)          # Index in name list
diff --git a/pypy/tool/pydis.py b/pypy/tool/pydis.py
deleted file mode 100644
--- a/pypy/tool/pydis.py
+++ /dev/null
@@ -1,202 +0,0 @@
-"""disassembler of Python byte code into mnemonics.
-
-XXX this only works for python-2.3 because of the linenumber 
-    optimization 
-
-"""
-
-import sys
-
-from pypy.tool import stdlib_opcode
-from pypy.tool.stdlib_opcode import *
-
-__all__ = ["dis","pydisassemble","distb","disco"] + stdlib_opcode.__all__
-
-EXTENDED_ARG = stdlib_opcode.opcodedesc.EXTENDED_ARG.index
-
-
-class Bytecode:
-    def __init__(self, disresult, bytecodeindex, oparg, lineno):
-        self.disresult = disresult 
-        self.index = bytecodeindex
-        self.op = ord(disresult.code.co_code[self.index])
-        self.name = opname[self.op]
-        self.oparg = oparg
-        self.lineno = lineno
-
-    def __eq__(self, other):
-        return (self.__class__ == other.__class__ and 
-                self.index == other.index and
-                self.op == other.op and
-                self.name == other.name and
-                self.oparg == other.oparg)
-
-    def __ne__(self, other):
-        return not (self == other)
-
-    def reprargstring(self, space = None):
-        """ return a string representation of any arguments. (empty for no args)"""
-        oparg = self.oparg
-        if oparg is None:
-            return ''
-        co = self.disresult.code
-        op = self.op
-        
-        s = repr(oparg).rjust(5) + " "
-        if op in hasconst:
-            consts = self.get_consts(space)
-            s += '(' + consts[oparg] + ')'
-        elif op in hasname:
-            s +=  '(' + co.co_names[oparg] + ')'
-        elif op in hasjrel:
-            s +=  '(to ' + repr(self.index + oparg) + ')'
-        elif op in haslocal:
-            s +=  '(' + co.co_varnames[oparg] + ')'
-        elif op in hascompare:
-            s +=  '(' + cmp_op[oparg] + ')'
-        elif op in hasfree:
-            #if free is None:
-            free = co.co_cellvars + co.co_freevars
-            s +=  '(' + free[oparg] + ')'
-        return s 
-
-    def get_consts(self, space=None):
-        # support both real code objects and PyCode objects
-        co = self.disresult.code
-        if hasattr(co, "co_consts"):
-            return [repr(c) for c in co.co_consts]
-
-        if space is None:
-            return [repr(c) for c in co.co_consts_w]
-        
-        r = lambda x: space.str_w(space.repr(x))
-        return [r(c) for c in co.co_consts_w]
-
-    def repr_with_space(self, space):
-        return self.name + self.reprargstring(space)
-
-    def __repr__(self):
-        return self.name + self.reprargstring()
-
-class DisResult:
-    """ an instance of this class gets returned for disassembling 
-        objects/functions/code objects whatever.    
-    """
-    def __init__(self, code):
-        self.code = code
-        self.bytecodes = []
-   
-    def append(self,  bytecodeindex, oparg, lineno):
-        """ append bytecode anaylsis information ..."""
-        bc = Bytecode(self, bytecodeindex, oparg, lineno)
-        self.bytecodes.append(bc)
-
-    def getbytecode(self, index):
-        """ return bytecode instance matching the given index. """
-        for bytecode in self.bytecodes:
-            if bytecode.index == index:
-                return bytecode
-        raise ValueError("no bytecode found on index %s in code \n%s" % (
-                index, pydis(self.code)))
-
-    def format(self):
-        lastlineno = -1
-        labels = findlabels(self.code.co_code)
-        lines = []
-        for bc in self.bytecodes:
-            l = []
-            if bc.lineno != lastlineno:
-                lastlineno = bc.lineno
-                l.append("%3d" % bc.lineno)
-            else:
-                l.append("   ")
-            l.append(bc.index in labels and ">>" or "  ")
-            l.append(repr(bc.index).rjust(4)) 
-            l.append(bc.name.ljust(20))
-            l.append(bc.reprargstring())
-            lines.append(" ".join(l))
-        return "\n".join(lines)
-
-    __repr__ = format
-
-def pydis(co): 
-    """return result of dissassembling a code object. """
-
-    if hasattr(co, 'func_code'):
-        co = co.func_code 
-
-    if hasattr(co, 'code'):
-        co = co.code 
-
-    disresult = DisResult(co)
-    code = co.co_code
-
-    byte_increments = [ord(c) for c in co.co_lnotab[0::2]]
-    line_increments = [ord(c) for c in co.co_lnotab[1::2]]
-    table_length = len(byte_increments)
-
-    lineno = co.co_firstlineno
-    table_index = 0
-    while (table_index < table_length
-           and byte_increments[table_index] == 0):
-        lineno += line_increments[table_index]
-        table_index += 1
-    addr = 0
-    line_incr = 0
-
-    n = len(code)
-    i = 0
-    extended_arg = 0
-    while i < n:
-        c = code[i]
-        op = ord(c)
-
-        if i >= addr:
-            lineno += line_incr
-            while table_index < table_length:
-                addr += byte_increments[table_index]
-                line_incr = line_increments[table_index]
-                table_index += 1
-                if line_incr:
-                    break
-            else:
-                addr = sys.maxint
-        current_bytecodeindex = i
-        i = i+1
-        oparg = None
-        if op >= HAVE_ARGUMENT:
-            oparg = ord(code[i]) + ord(code[i+1])*256 + extended_arg
-            extended_arg = 0
-            i = i+2
-            if op == EXTENDED_ARG:
-                extended_arg = oparg*65536L
-
-        disresult.append(current_bytecodeindex, oparg, lineno)
-    assert disresult is not None
-    return disresult
-
-def findlabels(code):
-    """Detect all offsets in a byte code which are jump targets.
-
-    Return the list of offsets.
-
-    """
-    labels = []
-    n = len(code)
-    i = 0
-    while i < n:
-        c = code[i]
-        op = ord(c)
-        i = i+1
-        if op >= HAVE_ARGUMENT:
-            oparg = ord(code[i]) + ord(code[i+1])*256
-            i = i+2

From pypy.commits at gmail.com  Sat Jul 28 09:23:54 2018
From: pypy.commits at gmail.com (cfbolz)
Date: Sat, 28 Jul 2018 06:23:54 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6: document merged branch
Message-ID: <5b5c6e6a.1c69fb81.3f712.5e5b@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: py3.6
Changeset: r94915:5e596e627b4e
Date: 2018-07-28 15:23 +0200
http://bitbucket.org/pypy/pypy/changeset/5e596e627b4e/

Log:	document merged branch

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -35,3 +35,7 @@
 .. branch: pyparser-improvements-3
 
 Small refactorings in the Python parser.
+
+.. branch: py3.6-wordcode
+
+implement new wordcode instruction encoding on the 3.6 branch

From pypy.commits at gmail.com  Sat Jul 28 09:27:19 2018
From: pypy.commits at gmail.com (AntoineDupre)
Date: Sat, 28 Jul 2018 06:27:19 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6: (duprean,
 antocuni) Test and fix for fstring exception message to mimic
 Cpython
Message-ID: <5b5c6f37.1c69fb81.26df5.9ba7@mx.google.com>

Author: Antoine Dupre <dupre.antoine.nicolas at gmail.com>
Branch: py3.6
Changeset: r94916:56432e26b463
Date: 2018-07-28 15:22 +0200
http://bitbucket.org/pypy/pypy/changeset/56432e26b463/

Log:	(duprean, antocuni) Test and fix for fstring exception message to
	mimic Cpython

diff --git a/pypy/objspace/std/newformat.py b/pypy/objspace/std/newformat.py
--- a/pypy/objspace/std/newformat.py
+++ b/pypy/objspace/std/newformat.py
@@ -583,7 +583,7 @@
 
         def _unknown_presentation(self, w_val):
             raise oefmt(self.space.w_ValueError,
-                        "unknown format code %s for object of type '%T'", self._type, w_val)
+                        "Unknown format code %s for object of type '%T'", self._type, w_val)
 
         def format_string(self, w_string):
             space = self.space
diff --git a/pypy/objspace/std/test/test_fsting.py b/pypy/objspace/std/test/test_fsting.py
new file mode 100644
--- /dev/null
+++ b/pypy/objspace/std/test/test_fsting.py
@@ -0,0 +1,8 @@
+class AppTestFstring:
+    def test_error_unknown_code(self):
+        """
+        def fn():
+            f'{1000:j}'
+        exc_info = raises(ValueError, fn)
+        assert str(exc_info.value).startswith("Unknown format code")
+        """

From pypy.commits at gmail.com  Sat Jul 28 09:27:22 2018
From: pypy.commits at gmail.com (AntoineDupre)
Date: Sat, 28 Jul 2018 06:27:22 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6: merge head
Message-ID: <5b5c6f3a.1c69fb81.4ef7f.28db@mx.google.com>

Author: Antoine Dupre <dupre.antoine.nicolas at gmail.com>
Branch: py3.6
Changeset: r94917:4a8044e8da87
Date: 2018-07-28 15:23 +0200
http://bitbucket.org/pypy/pypy/changeset/4a8044e8da87/

Log:	merge head

diff --git a/pypy/module/posix/test/test_scandir.py b/pypy/module/posix/test/test_scandir.py
--- a/pypy/module/posix/test/test_scandir.py
+++ b/pypy/module/posix/test/test_scandir.py
@@ -224,7 +224,7 @@
 
     def test_lstat(self):
         posix = self.posix
-        d = next(posix.scandir())
+        d = next(posix.scandir(self.dir1))
         with open(d) as fp:
             length = len(fp.read())
         assert posix.lstat(d).st_size == length

From pypy.commits at gmail.com  Sat Jul 28 09:27:24 2018
From: pypy.commits at gmail.com (AntoineDupre)
Date: Sat, 28 Jul 2018 06:27:24 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6: merge heads again
Message-ID: <5b5c6f3c.1c69fb81.a7e76.b8c0@mx.google.com>

Author: Antoine Dupre <dupre.antoine.nicolas at gmail.com>
Branch: py3.6
Changeset: r94918:5b1df32fb6f6
Date: 2018-07-28 15:25 +0200
http://bitbucket.org/pypy/pypy/changeset/5b1df32fb6f6/

Log:	merge heads again

diff too long, truncating to 2000 out of 2020 lines

diff --git a/lib-python/3/opcode.py b/lib-python/3/opcode.py
--- a/lib-python/3/opcode.py
+++ b/lib-python/3/opcode.py
@@ -216,6 +216,7 @@
 def_op('BUILD_SET_UNPACK', 153)
 
 def_op('FORMAT_VALUE', 155)   # in CPython 3.6, but available in PyPy from 3.5
+def_op('BUILD_CONST_KEY_MAP', 156)
 def_op('BUILD_STRING', 157)   # in CPython 3.6, but available in PyPy from 3.5
 
 # pypy modification, experimental bytecode
diff --git a/lib-python/3/test/test_opcodes.py b/lib-python/3/test/test_opcodes.py
--- a/lib-python/3/test/test_opcodes.py
+++ b/lib-python/3/test/test_opcodes.py
@@ -27,7 +27,9 @@
             with open(ann_module.__file__) as f:
                 txt = f.read()
             co = compile(txt, ann_module.__file__, 'exec')
-            self.assertEqual(co.co_firstlineno, 6)
+            # On PyPy, the lineno of multiline tokens is the *first* line, on
+            # CPython the last (CPython expects 6 here)
+            self.assertEqual(co.co_firstlineno, 3)
         except OSError:
             pass
 
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -35,3 +35,7 @@
 .. branch: pyparser-improvements-3
 
 Small refactorings in the Python parser.
+
+.. branch: py3.6-wordcode
+
+implement new wordcode instruction encoding on the 3.6 branch
diff --git a/pypy/interpreter/astcompiler/assemble.py b/pypy/interpreter/astcompiler/assemble.py
--- a/pypy/interpreter/astcompiler/assemble.py
+++ b/pypy/interpreter/astcompiler/assemble.py
@@ -21,6 +21,8 @@
     def __init__(self, opcode, arg=0):
         self.opcode = opcode
         self.arg = arg
+        if opcode < ops.HAVE_ARGUMENT:
+            assert arg == 0
         self.lineno = 0
         self.has_jump = False
 
@@ -28,9 +30,33 @@
         """Return the size of bytes of this instruction when it is
         encoded.
         """
-        if self.opcode >= ops.HAVE_ARGUMENT:
-            return (6 if self.arg > 0xFFFF else 3)
-        return 1
+        if self.arg <= 0xff:
+            return 2
+        if self.arg <= 0xffff:
+            return 4
+        if self.arg <= 0xffffff:
+            return 6
+        return 8
+
+    def encode(self, code):
+        opcode = self.opcode
+
+        arg = self.arg
+        size = self.size()
+        if size == 8:
+            code.append(chr(ops.EXTENDED_ARG))
+            code.append(chr((arg >> 24) & 0xff))
+            assert ((arg >> 24) & 0xff) == (arg >> 24)
+        if size >= 6:
+            code.append(chr(ops.EXTENDED_ARG))
+            code.append(chr((arg >> 16) & 0xff))
+        if size >= 4:
+            code.append(chr(ops.EXTENDED_ARG))
+            code.append(chr((arg >> 8) & 0xff))
+        if size >= 2:
+            code.append(chr(opcode))
+            code.append(chr(arg & 0xff))
+
 
     def jump_to(self, target, absolute=False):
         """Indicate the target this jump instruction.
@@ -121,20 +147,9 @@
         """Encode the instructions in this block into bytecode."""
         code = []
         for instr in self.instructions:
-            opcode = instr.opcode
-            if opcode >= ops.HAVE_ARGUMENT:
-                arg = instr.arg
-                if instr.arg > 0xFFFF:
-                    ext = arg >> 16
-                    code.append(chr(ops.EXTENDED_ARG))
-                    code.append(chr(ext & 0xFF))
-                    code.append(chr(ext >> 8))
-                    arg &= 0xFFFF
-                code.append(chr(opcode))
-                code.append(chr(arg & 0xFF))
-                code.append(chr(arg >> 8))
-            else:
-                code.append(chr(opcode))
+            instr.encode(code)
+        assert len(code) == self.code_size()
+        assert len(code) & 1 == 0
         return ''.join(code)
 
 
@@ -185,6 +200,13 @@
         self.lineno = 0
         self.add_none_to_final_return = True
 
+    def _check_consistency(self, blocks):
+        current_off = 0
+        for block in blocks:
+            assert block.offset == current_off
+            for instr in block.instructions:
+                current_off += instr.size()
+
     def new_block(self):
         return Block()
 
@@ -315,14 +337,12 @@
 
     def _resolve_block_targets(self, blocks):
         """Compute the arguments of jump instructions."""
-        last_extended_arg_count = 0
         # The reason for this loop is extended jumps.  EXTENDED_ARG
-        # extends the bytecode size, so it might invalidate the offsets
-        # we've already given.  Thus we have to loop until the number of
-        # extended args is stable.  Any extended jump at all is
-        # extremely rare, so performance is not too concerning.
+        # extends the bytecode size, so it might invalidate the offsets we've
+        # already given.  Thus we have to loop until the size of all jump
+        # instructions is stable. Any extended jump at all is extremely rare,
+        # so performance should not be too concerning.
         while True:
-            extended_arg_count = 0
             offset = 0
             force_redo = False
             # Calculate the code offset of each block.
@@ -332,7 +352,8 @@
             for block in blocks:
                 offset = block.offset
                 for instr in block.instructions:
-                    offset += instr.size()
+                    size = instr.size()
+                    offset += size
                     if instr.has_jump:
                         target, absolute = instr.jump
                         op = instr.opcode
@@ -351,22 +372,21 @@
                                     instr.opcode = ops.RETURN_VALUE
                                     instr.arg = 0
                                     instr.has_jump = False
-                                    # The size of the code changed,
+                                    # The size of the code maybe have changed,
                                     # we have to trigger another pass
-                                    force_redo = True
+                                    if instr.size() != size:
+                                        force_redo = True
                                     continue
                         if absolute:
                             jump_arg = target.offset
                         else:
                             jump_arg = target.offset - offset
                         instr.arg = jump_arg
-                        if jump_arg > 0xFFFF:
-                            extended_arg_count += 1
-            if (extended_arg_count == last_extended_arg_count and
-                not force_redo):
-                break
-            else:
-                last_extended_arg_count = extended_arg_count
+                        if instr.size() != size:
+                            force_redo = True
+            if not force_redo:
+                self._check_consistency(blocks)
+                return
 
     def _build_consts_array(self):
         """Turn the applevel constants dictionary into a list."""
@@ -738,7 +758,7 @@
     return -2 - _num_args(arg) - ((arg >> 16) & 0xFFFF)
 
 def _compute_MAKE_FUNCTION(arg):
-    return -1 - _num_args(arg) - ((arg >> 16) & 0xFFFF)
+    return -1 - bool(arg & 0x01) - bool(arg & 0x02) - bool(arg & 0x04) - bool(arg & 0x08)
 
 def _compute_BUILD_SLICE(arg):
     if arg == 3:
@@ -775,6 +795,9 @@
 def _compute_BUILD_STRING(arg):
     return 1 - arg
 
+def _compute_BUILD_CONST_KEY_MAP(arg):
+    return  -arg
+
 
 _stack_effect_computers = {}
 for name, func in globals().items():
diff --git a/pypy/interpreter/astcompiler/codegen.py b/pypy/interpreter/astcompiler/codegen.py
--- a/pypy/interpreter/astcompiler/codegen.py
+++ b/pypy/interpreter/astcompiler/codegen.py
@@ -307,7 +307,7 @@
         # if the scope contained an annotated variable assignemt,
         # this will emit the requisite SETUP_ANNOTATIONS
         if self.scope.contains_annotated and not isinstance(self, AbstractFunctionCodeGenerator):
-            self.emit_op(ops.SETUP_ANNOTATIONS)
+            return self.emit_op(ops.SETUP_ANNOTATIONS)
 
     def visit_Module(self, mod):
         if not self._handle_body(mod.body):
@@ -321,10 +321,11 @@
         self.add_none_to_final_return = False
         mod.body.walkabout(self)
 
-    def _make_function(self, code, num_defaults=0, qualname=None):
+    def _make_function(self, code, oparg=0, qualname=None):
         """Emit the opcodes to turn a code object into a function."""
         w_qualname = self.space.newtext(qualname or code.co_name)
         if code.co_freevars:
+            oparg = oparg | 0x08
             # Load cell and free vars to pass on.
             for free in code.co_freevars:
                 free_scope = self.scope.lookup(free)
@@ -335,24 +336,25 @@
                     index = self.free_vars[free]
                 self.emit_op_arg(ops.LOAD_CLOSURE, index)
             self.emit_op_arg(ops.BUILD_TUPLE, len(code.co_freevars))
-            self.load_const(code)
-            self.load_const(w_qualname)
-            self.emit_op_arg(ops.MAKE_CLOSURE, num_defaults)
-        else:
-            self.load_const(code)
-            self.load_const(w_qualname)
-            self.emit_op_arg(ops.MAKE_FUNCTION, num_defaults)
+        self.load_const(code)
+        self.load_const(w_qualname)
+        self.emit_op_arg(ops.MAKE_FUNCTION, oparg)
 
     def _visit_kwonlydefaults(self, args):
         defaults = 0
+        keys_w = []
         for i, default in enumerate(args.kw_defaults):
             if default:
                 kwonly = args.kwonlyargs[i]
                 assert isinstance(kwonly, ast.arg)
                 mangled = self.scope.mangle(kwonly.arg)
-                self.load_const(self.space.newtext(mangled))
+                keys_w.append(self.space.newtext(mangled))
                 default.walkabout(self)
                 defaults += 1
+        if keys_w:
+            w_tup = self.space.newtuple(keys_w)
+            self.load_const(w_tup)
+            self.emit_op_arg(ops.BUILD_CONST_KEY_MAP, len(keys_w))
         return defaults
 
     def _visit_arg_annotation(self, name, ann, names):
@@ -387,7 +389,7 @@
                 self.error("too many annotations", func)
             w_tup = space.newtuple([space.newtext(name) for name in names])
             self.load_const(w_tup)
-            l += 1
+            self.emit_op_arg(ops.BUILD_CONST_KEY_MAP, l)
         return l
 
     @specialize.arg(2)
@@ -396,16 +398,25 @@
         # Load decorators first, but apply them after the function is created.
         self.visit_sequence(func.decorator_list)
         args = func.args
+
         assert isinstance(args, ast.arguments)
+
+        oparg = 0
         self.visit_sequence(args.defaults)
-        kw_default_count = 0
+
+        if args.defaults is not None and len(args.defaults):
+            oparg = oparg | 0x01
+            self.emit_op_arg(ops.BUILD_TUPLE, len(args.defaults))
+
         if args.kwonlyargs:
             kw_default_count = self._visit_kwonlydefaults(args)
+            if kw_default_count:
+                oparg = oparg | 0x02
+
         num_annotations = self._visit_annotations(func, args, func.returns)
-        num_defaults = len(args.defaults) if args.defaults is not None else 0
-        oparg = num_defaults
-        oparg |= kw_default_count << 8
-        oparg |= num_annotations << 16
+        if num_annotations:
+            oparg = oparg | 0x04
+
         code, qualname = self.sub_scope(function_code_generator, func.name,
                                         func, func.lineno)
         self._make_function(code, oparg, qualname=qualname)
@@ -425,15 +436,20 @@
         self.update_position(lam.lineno)
         args = lam.args
         assert isinstance(args, ast.arguments)
+
         self.visit_sequence(args.defaults)
-        kw_default_count = 0
+
+        oparg = 0
+        if args.defaults is not None and len(args.defaults):
+            oparg = oparg | 0x01
+            self.emit_op_arg(ops.BUILD_TUPLE, len(args.defaults))
+
         if args.kwonlyargs:
             kw_default_count = self._visit_kwonlydefaults(args)
-        default_count = len(args.defaults) if args.defaults is not None else 0
+            if kw_default_count:
+                oparg = oparg | 0x02
         code, qualname = self.sub_scope(
             LambdaCodeGenerator, "<lambda>", lam, lam.lineno)
-        oparg = default_count
-        oparg |= kw_default_count << 8
         self._make_function(code, oparg, qualname=qualname)
 
     def visit_ClassDef(self, cls):
@@ -1281,28 +1297,46 @@
         containers = 0
         elements = 0
         is_unpacking = False
+        all_constant_keys_w = None
         if d.values:
+            if len(d.keys) < 0xffff:
+                all_constant_keys_w = []
+                for key in d.keys:
+                    if key is None or key.as_constant() is None:
+                        all_constant_keys_w = None
+                        break
+                    else:
+                        all_constant_keys_w.append(key.as_constant())
             for i in range(len(d.values)):
                 key = d.keys[i]
                 is_unpacking = key is None
                 if elements == 0xFFFF or (elements and is_unpacking):
+                    assert all_constant_keys_w is None
                     self.emit_op_arg(ops.BUILD_MAP, elements)
                     containers += 1
                     elements = 0
                 if is_unpacking:
+                    assert all_constant_keys_w is None
                     d.values[i].walkabout(self)
                     containers += 1
                 else:
-                    key.walkabout(self)
+                    if not all_constant_keys_w:
+                        key.walkabout(self)
                     d.values[i].walkabout(self)
                     elements += 1
         if elements or containers == 0:
-            self.emit_op_arg(ops.BUILD_MAP, elements)
-            containers += 1
+            if all_constant_keys_w:
+                w_tup = self.space.newtuple(all_constant_keys_w)
+                self.load_const(w_tup)
+                self.emit_op_arg(ops.BUILD_CONST_KEY_MAP, elements)
+            else:
+                self.emit_op_arg(ops.BUILD_MAP, elements)
+                containers += 1
         # If there is more than one dict, they need to be merged into
         # a new dict. If there is one dict and it's an unpacking, then
         #it needs to be copied into a new dict.
         while containers > 1 or is_unpacking:
+            assert all_constant_keys_w is None
             oparg = min(containers, 255)
             self.emit_op_arg(ops.BUILD_MAP_UNPACK, oparg)
             containers -= (oparg - 1)
@@ -1699,6 +1733,12 @@
                                      symbols, compile_info, qualname=None)
 
     def _compile(self, tree):
+        if isinstance(tree, ast.Module):
+            if tree.body:
+                self.first_lineno = tree.body[0].lineno
+            else:
+                self.first_lineno = self.lineno = 1
+
         self._maybe_setup_annotations()
         tree.walkabout(self)
 
diff --git a/pypy/interpreter/astcompiler/test/test_astbuilder.py b/pypy/interpreter/astcompiler/test/test_astbuilder.py
--- a/pypy/interpreter/astcompiler/test/test_astbuilder.py
+++ b/pypy/interpreter/astcompiler/test/test_astbuilder.py
@@ -939,7 +939,7 @@
 
     def test_flufl(self):
         source = "x <> y"
-        raises(SyntaxError, self.get_ast, source)
+        py.test.raises(SyntaxError, self.get_ast, source)
         comp = self.get_first_expr(source,
                                    flags=consts.CO_FUTURE_BARRY_AS_BDFL)
         assert isinstance(comp, ast.Compare)
@@ -1167,7 +1167,7 @@
         s = self.get_first_expr("b'hi' b' implicitly' b' extra'")
         assert isinstance(s, ast.Bytes)
         assert space.eq_w(s.s, space.newbytes("hi implicitly extra"))
-        raises(SyntaxError, self.get_first_expr, "b'hello' 'world'")
+        py.test.raises(SyntaxError, self.get_first_expr, "b'hello' 'world'")
         sentence = u"Die M&#228;nner &#228;rgern sich!"
         source = u"# coding: utf-7\nstuff = '%s'" % (sentence,)
         info = pyparse.CompileInfo("<test>", "exec")
@@ -1362,8 +1362,8 @@
         assert isinstance(if2, ast.Name)
 
     def test_cpython_issue12983(self):
-        raises(SyntaxError, self.get_ast, r"""b'\x'""")
-        raises(SyntaxError, self.get_ast, r"""b'\x0'""")
+        py.test.raises(SyntaxError, self.get_ast, r"""b'\x'""")
+        py.test.raises(SyntaxError, self.get_ast, r"""b'\x0'""")
 
     def test_matmul(self):
         mod = self.get_ast("a @ b")
diff --git a/pypy/interpreter/astcompiler/test/test_compiler.py b/pypy/interpreter/astcompiler/test/test_compiler.py
--- a/pypy/interpreter/astcompiler/test/test_compiler.py
+++ b/pypy/interpreter/astcompiler/test/test_compiler.py
@@ -72,10 +72,9 @@
     def check(self, w_dict, evalexpr, expected):
         # for now, we compile evalexpr with CPython's compiler but run
         # it with our own interpreter to extract the data from w_dict
-        co_expr = compile(evalexpr, '<evalexpr>', 'eval')
         space = self.space
-        pyco_expr = PyCode._from_code(space, co_expr)
-        w_res = pyco_expr.exec_host_bytecode(w_dict, w_dict)
+        pyco_expr = space.createcompiler().compile(evalexpr, '<evalexpr>', 'eval', 0)
+        w_res = space.exec_(pyco_expr, w_dict, w_dict)
         res = space.str_w(space.repr(w_res))
         expected_repr = self.get_py3_repr(expected)
         if isinstance(expected, float):
@@ -1011,6 +1010,7 @@
         ("C4.__doc__", 'docstring'),
         ("C4.__doc__", 'docstring'),
         ("__doc__", None),])
+
     def test_remove_docstring(self, expr, result):
         source = '"module_docstring"\n' + """if 1:
         def f1():
@@ -1210,12 +1210,12 @@
         yield self.st, """z=f'{f"{0}"*3}'""", 'z', '000'
 
     def test_fstring_error(self):
-        raises(SyntaxError, self.run, "f'{}'")
-        raises(SyntaxError, self.run, "f'{   \t   }'")
-        raises(SyntaxError, self.run, "f'{5#}'")
-        raises(SyntaxError, self.run, "f'{5)#}'")
-        raises(SyntaxError, self.run, "f'''{5)\n#}'''")
-        raises(SyntaxError, self.run, "f'\\x'")
+        py.test.raises(SyntaxError, self.run, "f'{}'")
+        py.test.raises(SyntaxError, self.run, "f'{   \t   }'")
+        py.test.raises(SyntaxError, self.run, "f'{5#}'")
+        py.test.raises(SyntaxError, self.run, "f'{5)#}'")
+        py.test.raises(SyntaxError, self.run, "f'''{5)\n#}'''")
+        py.test.raises(SyntaxError, self.run, "f'\\x'")
 
     def test_fstring_encoding(self):
         src = """# -*- coding: latin-1 -*-\nz=ord(f'{"\xd8"}')\n"""
@@ -1512,3 +1512,18 @@
             del []
         """
         generate_function_code(source, self.space)
+
+    def test_make_constant_map(self):
+        source = """def f():
+            return {"A": 1, "b": 2}
+        """
+        counts = self.count_instructions(source)
+        assert ops.BUILD_MAP not in counts
+        source = """def f():
+            return {"a": 1, "b": {}, 1: {"a": x}}
+        """
+        counts = self.count_instructions(source)
+        assert counts[ops.BUILD_MAP] == 1 # the empty dict
+        assert counts[ops.BUILD_CONST_KEY_MAP] == 2
+
+
diff --git a/pypy/interpreter/astcompiler/test/test_misc.py b/pypy/interpreter/astcompiler/test/test_misc.py
--- a/pypy/interpreter/astcompiler/test/test_misc.py
+++ b/pypy/interpreter/astcompiler/test/test_misc.py
@@ -1,4 +1,5 @@
 from pypy.interpreter.astcompiler.misc import mangle
+from pypy.interpreter.astcompiler.assemble import Instruction, ops
 
 def test_mangle():
     assert mangle("foo", "Bar") == "foo"
@@ -13,6 +14,34 @@
     assert mangle("__foo", "___") == "__foo"
     assert mangle("___foo", "__Bar") == "_Bar___foo"
 
+def test_instruction_size():
+    assert Instruction(ops.POP_TOP).size() == 2
+    assert Instruction(ops.LOAD_FAST, 23).size() == 2
+    assert Instruction(ops.LOAD_FAST, 0xfff0).size() == 4
+    assert Instruction(ops.LOAD_FAST, 0x10000).size() == 6
+    assert Instruction(ops.LOAD_FAST, 0x1000000).size() == 8
+
+def test_instruction_encode():
+    c = []
+    Instruction(ops.POP_TOP).encode(c)
+    assert c == [chr(ops.POP_TOP), '\x00']
+
+    c = []
+    Instruction(ops.LOAD_FAST, 1).encode(c)
+    assert c == [chr(ops.LOAD_FAST), '\x01']
+
+    c = []
+    Instruction(ops.LOAD_FAST, 0x201).encode(c)
+    assert c == [chr(ops.EXTENDED_ARG), '\x02', chr(ops.LOAD_FAST), '\x01']
+
+    c = []
+    Instruction(ops.LOAD_FAST, 0x30201).encode(c)
+    assert c == [chr(ops.EXTENDED_ARG), '\x03', chr(ops.EXTENDED_ARG), '\x02', chr(ops.LOAD_FAST), '\x01']
+
+    c = []
+    Instruction(ops.LOAD_FAST, 0x5030201).encode(c)
+    assert c == [chr(ops.EXTENDED_ARG), '\x05', chr(ops.EXTENDED_ARG), '\x03', chr(ops.EXTENDED_ARG), '\x02', chr(ops.LOAD_FAST), '\x01']
+
 def app_test_warning_to_error_translation():
     import warnings
 
diff --git a/pypy/interpreter/generator.py b/pypy/interpreter/generator.py
--- a/pypy/interpreter/generator.py
+++ b/pypy/interpreter/generator.py
@@ -166,16 +166,20 @@
             # Normal case: the call above raises Yield.
             # We reach this point if the iterable is exhausted.
             last_instr = jit.promote(frame.last_instr)
+            assert last_instr & 1 == 0
             assert last_instr >= 0
-            return r_uint(last_instr + 1)
+            return r_uint(last_instr + 2)
 
         if isinstance(w_arg_or_err, SApplicationException):
             return frame.handle_generator_error(w_arg_or_err.operr)
 
         last_instr = jit.promote(frame.last_instr)
         if last_instr != -1:
+            assert last_instr & 1 == 0
             frame.pushvalue(w_arg_or_err)
-        return r_uint(last_instr + 1)
+            return r_uint(last_instr + 2)
+        else:
+            return r_uint(0)
 
     def next_yield_from(self, frame, w_yf, w_inputvalue_or_err):
         """Fetch the next item of the current 'yield from', push it on
diff --git a/pypy/interpreter/interactive.py b/pypy/interpreter/interactive.py
--- a/pypy/interpreter/interactive.py
+++ b/pypy/interpreter/interactive.py
@@ -213,7 +213,7 @@
             ec.bytecode_only_trace = self._orig_bytecode_only_trace
 
     def _do_bytecode_only_trace(self, frame):
-        from pypy.tool.pydis import Bytecode, HAVE_ARGUMENT
+        from pypy.tool import opcode3, dis3
 
         if frame.hide():
             return
@@ -221,18 +221,12 @@
         self.unsettrace()
         next_instr = frame.last_instr
         opcode = ord(frame.pycode.co_code[next_instr])
+        oparg = ord(frame.pycode.co_code[next_instr+1])
 
-        oparg = 0
-        if opcode >= HAVE_ARGUMENT:
-            lo = ord(frame.pycode.co_code[next_instr+1])
-            hi = ord(frame.pycode.co_code[next_instr+2])
-            oparg = (hi * 256) | lo
-
-        class fake:
-            code = frame.pycode
-        bytecode = Bytecode(fake, next_instr, oparg, 0)
+        argrepr = reprargstring(self.space, frame.pycode, opcode, oparg)
+        oprepr = opcode3.opname[opcode] + argrepr.ljust(5)
         print '\t%-19s %s' % (str(frame.pycode.co_name) + ':',
-                              bytecode.repr_with_space(self.space))
+                              oprepr)
         self.settrace()
 
     def checktrace(self):
@@ -255,3 +249,26 @@
 
 class IncompleteInput(Exception):
     pass
+
+
+def reprargstring(space, pycode, opcode, oparg):
+    """ return a string representation of any arguments. (empty for no args)"""
+    from pypy.tool import opcode3
+    if oparg is None:
+        return ''
+    s = repr(oparg).rjust(5) + " "
+    if opcode in opcode3.hasconst:
+        r = space.text_w(space.repr(pycode.co_consts_w[oparg]))
+        s += '(' + r + ')'
+    elif opcode in opcode3.hasname:
+        s +=  '(' + pycode.co_names[oparg] + ')'
+    elif opcode in opcode3.hasjrel:
+        s +=  '(to ' + repr(self.index + oparg) + ')'
+    elif opcode in opcode3.haslocal:
+        s +=  '(' + pycode.co_varnames[oparg] + ')'
+    elif opcode in opcode3.hascompare:
+        s +=  '(' + opcode3.cmp_op[oparg] + ')'
+    elif opcode in opcode3.hasfree:
+        free = pycode.co_cellvars + pycode.co_freevars
+        s +=  '(' + free[oparg] + ')'
+    return s
diff --git a/pypy/interpreter/pycode.py b/pypy/interpreter/pycode.py
--- a/pypy/interpreter/pycode.py
+++ b/pypy/interpreter/pycode.py
@@ -39,7 +39,7 @@
 # time you make pyc files incompatible.  This value ends up in the frozen
 # importlib, via MAGIC_NUMBER in module/_frozen_importlib/__init__.
 
-pypy_incremental_magic = 128 # bump it by 16
+pypy_incremental_magic = 160 # bump it by 16
 assert pypy_incremental_magic % 16 == 0
 assert pypy_incremental_magic < 3000 # the magic number of Python 3. There are
                                      # no known magic numbers below this value
@@ -216,6 +216,7 @@
         """
         Hack to initialize the code object from a real (CPython) one.
         """
+        raise TypeError("assert reinterpretation for applevel tests is broken on PyPy3!")
         assert isinstance(code, types.CodeType)
         newconsts_w = [None] * len(code.co_consts)
         num = 0
@@ -301,11 +302,7 @@
                 w_co.remove_docstrings(space)
 
     def exec_host_bytecode(self, w_globals, w_locals):
-        if sys.version_info < (2, 7):
-            raise Exception("PyPy no longer supports Python 2.6 or lower")
-        frame = self.space.FrameClass(self.space, self, w_globals, None)
-        frame.setdictscope(w_locals)
-        return frame.run()
+        raise Exception("no longer supported after the switch to wordcode!")
 
     def dump(self):
         """NOT_RPYTHON: A dis.dis() dump of the code object."""
diff --git a/pypy/interpreter/pyframe.py b/pypy/interpreter/pyframe.py
--- a/pypy/interpreter/pyframe.py
+++ b/pypy/interpreter/pyframe.py
@@ -691,6 +691,7 @@
         endblock = [-1]     # current finally/except block stack
         addr = 0
         while addr < len(code):
+            assert addr & 1 == 0
             op = ord(code[addr])
             if op in (SETUP_LOOP, SETUP_EXCEPT, SETUP_FINALLY, SETUP_WITH,
                       SETUP_ASYNC_WITH):
@@ -713,10 +714,7 @@
             if addr == self.last_instr:
                 f_lasti_handler_addr = endblock[-1]
 
-            if op >= HAVE_ARGUMENT:
-                addr += 3
-            else:
-                addr += 1
+            addr += 2
 
         if len(blockstack) != 0 or len(endblock) != 1:
             raise oefmt(space.w_SystemError,
@@ -774,6 +772,7 @@
             block.cleanupstack(self)
 
         self.getorcreatedebug().f_lineno = new_lineno
+        assert new_lasti & 1 == 0
         self.last_instr = new_lasti
 
     def get_last_lineno(self):
diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py
--- a/pypy/interpreter/pyopcode.py
+++ b/pypy/interpreter/pyopcode.py
@@ -62,6 +62,7 @@
         co_code = pycode.co_code
         try:
             while True:
+                assert next_instr & 1 == 0
                 next_instr = self.handle_bytecode(co_code, next_instr, ec)
         except ExitFrame:
             return self.popvalue()
@@ -152,22 +153,17 @@
     @jit.unroll_safe
     def dispatch_bytecode(self, co_code, next_instr, ec):
         while True:
+            assert next_instr & 1 == 0
             self.last_instr = intmask(next_instr)
             if jit.we_are_jitted():
                 ec.bytecode_only_trace(self)
             else:
                 ec.bytecode_trace(self)
             next_instr = r_uint(self.last_instr)
+            assert next_instr & 1 == 0
             opcode = ord(co_code[next_instr])
-            next_instr += 1
-
-            if opcode >= HAVE_ARGUMENT:
-                lo = ord(co_code[next_instr])
-                hi = ord(co_code[next_instr+1])
-                next_instr += 2
-                oparg = (hi * 256) | lo
-            else:
-                oparg = 0
+            oparg = ord(co_code[next_instr + 1])
+            next_instr += 2
 
             # note: the structure of the code here is such that it makes
             # (after translation) a big "if/elif" chain, which is then
@@ -175,12 +171,11 @@
 
             while opcode == opcodedesc.EXTENDED_ARG.index:
                 opcode = ord(co_code[next_instr])
+                arg = ord(co_code[next_instr + 1])
                 if opcode < HAVE_ARGUMENT:
                     raise BytecodeCorruption
-                lo = ord(co_code[next_instr+1])
-                hi = ord(co_code[next_instr+2])
-                next_instr += 3
-                oparg = (oparg * 65536) | (hi * 256) | lo
+                next_instr += 2
+                oparg = (oparg * 256) | arg
 
             if opcode == opcodedesc.RETURN_VALUE.index:
                 w_returnvalue = self.popvalue()
@@ -252,6 +247,8 @@
                 self.BINARY_TRUE_DIVIDE(oparg, next_instr)
             elif opcode == opcodedesc.BINARY_XOR.index:
                 self.BINARY_XOR(oparg, next_instr)
+            elif opcode == opcodedesc.BUILD_CONST_KEY_MAP.index:
+                self.BUILD_CONST_KEY_MAP(oparg, next_instr)
             elif opcode == opcodedesc.BUILD_LIST.index:
                 self.BUILD_LIST(oparg, next_instr)
             elif opcode == opcodedesc.BUILD_LIST_FROM_ARG.index:
@@ -362,8 +359,6 @@
                 self.LOAD_NAME(oparg, next_instr)
             elif opcode == opcodedesc.LOOKUP_METHOD.index:
                 self.LOOKUP_METHOD(oparg, next_instr)
-            elif opcode == opcodedesc.MAKE_CLOSURE.index:
-                self.MAKE_CLOSURE(oparg, next_instr)
             elif opcode == opcodedesc.MAKE_FUNCTION.index:
                 self.MAKE_FUNCTION(oparg, next_instr)
             elif opcode == opcodedesc.MAP_ADD.index:
@@ -1357,47 +1352,42 @@
         self.call_function(oparg, w_varkw, has_vararg=True)
 
     @jit.unroll_safe
-    def _make_function(self, oparg, freevars=None):
+    def MAKE_FUNCTION(self, oparg, next_instr):
         space = self.space
         w_qualname = self.popvalue()
         qualname = self.space.unicode_w(w_qualname)
         w_codeobj = self.popvalue()
         codeobj = self.space.interp_w(PyCode, w_codeobj)
-        if freevars is not None:
-            # Pop freevars
-            self.popvalue()
-        posdefaults = oparg & 0xFF
-        kwdefaults = (oparg >> 8) & 0xFF
-        num_annotations = (oparg >> 16) & 0xFF
-        w_ann = None
-        if num_annotations:
-            names_w = space.fixedview(self.popvalue())
-            w_ann = space.newdict(strdict=True)
-            for i in range(len(names_w) - 1, -1, -1):
-                space.setitem(w_ann, names_w[i], self.popvalue())
-        kw_defs_w = None
-        if kwdefaults:
-            kw_defs_w = []
-            for i in range(kwdefaults):
-                w_defvalue = self.popvalue()
-                w_defname = self.popvalue()
-                kw_defs_w.append((w_defname, w_defvalue))
-        defaultarguments = self.popvalues(posdefaults)
+        assert 0 <= oparg <= 0x0F
+        if oparg & 0x08:
+            w_freevarstuple = self.popvalue()
+            # XXX this list copy is expensive, it's purely for the annotator
+            freevars = [self.space.interp_w(Cell, cell)
+                        for cell in self.space.fixedview(w_freevarstuple)]
+        else:
+            freevars = None
+        if oparg & 0x04:
+            w_ann = self.popvalue()
+        else:
+            w_ann = None
+        if oparg & 0x02:
+            w_kw_defs = self.popvalue()
+            # XXX
+            kw_defs_w = [space.unpackiterable(w_tup)
+                            for w_tup in space.fixedview(
+                                space.call_method(w_kw_defs, 'items'))]
+        else:
+            kw_defs_w = None
+        if oparg & 0x01:
+            defaultarguments = space.fixedview(self.popvalue())
+        else:
+            defaultarguments = []
+
         fn = function.Function(space, codeobj, self.get_w_globals(),
                                defaultarguments,
                                kw_defs_w, freevars, w_ann, qualname=qualname)
         self.pushvalue(fn)
 
-    def MAKE_FUNCTION(self, oparg, next_instr):
-        return self._make_function(oparg)
-
-    @jit.unroll_safe
-    def MAKE_CLOSURE(self, oparg, next_instr):
-        w_freevarstuple = self.peekvalue(2)
-        freevars = [self.space.interp_w(Cell, cell)
-                    for cell in self.space.fixedview(w_freevarstuple)]
-        self._make_function(oparg, freevars)
-
     def BUILD_SLICE(self, numargs, next_instr):
         if numargs == 3:
             w_step = self.popvalue()
@@ -1447,7 +1437,18 @@
             w_value = self.peekvalue(2 * i)
             w_key = self.peekvalue(2 * i + 1)
             self.space.setitem(w_dict, w_key, w_value)
-        self.popvalues(2 * itemcount)
+        self.dropvalues(2 * itemcount)
+        self.pushvalue(w_dict)
+
+    @jit.unroll_safe
+    def BUILD_CONST_KEY_MAP(self, itemcount, next_instr):
+        keys_w = self.space.fixedview(self.popvalue())
+        w_dict = self.space.newdict()
+        for i in range(itemcount):
+            w_value = self.peekvalue(itemcount - 1 - i)
+            w_key = keys_w[i]
+            self.space.setitem(w_dict, w_key, w_value)
+        self.dropvalues(itemcount)
         self.pushvalue(w_dict)
 
     @jit.unroll_safe
@@ -1456,7 +1457,7 @@
         for i in range(itemcount-1, -1, -1):
             w_item = self.peekvalue(i)
             self.space.call_method(w_set, 'add', w_item)
-        self.popvalues(itemcount)
+        self.dropvalues(itemcount)
         self.pushvalue(w_set)
 
     @jit.unroll_safe
diff --git a/pypy/interpreter/test/test_annotations.py b/pypy/interpreter/test/test_annotations.py
--- a/pypy/interpreter/test/test_annotations.py
+++ b/pypy/interpreter/test/test_annotations.py
@@ -132,3 +132,11 @@
             ''')
         """)
 
+    def test_lineno(self):
+        s = """
+
+a: int
+        """
+        c = compile(s, "f", "exec")
+        assert c.co_firstlineno == 3
+
diff --git a/pypy/interpreter/test/test_function.py b/pypy/interpreter/test/test_function.py
--- a/pypy/interpreter/test/test_function.py
+++ b/pypy/interpreter/test/test_function.py
@@ -643,11 +643,20 @@
 
 
 class TestMethod:
-    def setup_method(self, method):
-        def c(self, bar):
-            return bar
-        code = PyCode._from_code(self.space, c.__code__)
-        self.fn = Function(self.space, code, self.space.newdict())
+    @classmethod
+    def compile(cls, src):
+        assert src.strip().startswith("def ")
+        compiler = cls.space.createcompiler()
+        code = compiler.compile(src, '<hello>', 'exec', 0).co_consts_w[0]
+        return Function(cls.space, code, cls.space.newdict())
+
+    def setup_class(cls):
+        src = """
+def c(self, bar):
+    return bar
+        """
+        cls.fn = cls.compile(src)
+
 
     def test_get(self):
         space = self.space
@@ -672,9 +681,7 @@
     def test_method_get(self):
         space = self.space
         # Create some function for this test only
-        def m(self): return self
-        func = Function(space, PyCode._from_code(self.space, m.__code__),
-                        space.newdict())
+        func = self.compile("def m(self): return self")
         # Some shorthands
         obj1 = space.wrap(23)
         obj2 = space.wrap(42)
@@ -696,6 +703,11 @@
         assert meth3 is func
 
 class TestShortcuts(object):
+    def compile(self, src):
+        assert src.strip().startswith("def ")
+        compiler = self.space.createcompiler()
+        code = compiler.compile(src, '<hello>', 'exec', 0).co_consts_w[0]
+        return Function(self.space, code, self.space.newdict())
 
     def test_call_function(self):
         space = self.space
@@ -703,14 +715,15 @@
         d = {}
         for i in range(10):
             args = "(" + ''.join(["a%d," % a for a in range(i)]) + ")"
-            exec """
+            src = """
 def f%s:
     return %s
-""" % (args, args) in d
+""" % (args, args)
+            exec src in d
             f = d['f']
             res = f(*range(i))
-            code = PyCode._from_code(self.space, f.__code__)
-            fn = Function(self.space, code, self.space.newdict())
+            fn = self.compile(src)
+            code = fn.code
 
             assert fn.code.fast_natural_arity == i|PyCode.FLATPYCALL
             if i < 5:
@@ -729,18 +742,18 @@
     def test_flatcall(self):
         space = self.space
 
-        def f(a):
-            return a
-        code = PyCode._from_code(self.space, f.__code__)
-        fn = Function(self.space, code, self.space.newdict())
+        src = """
+def f(a):
+    return a"""
+        fn = self.compile(src)
 
         assert fn.code.fast_natural_arity == 1|PyCode.FLATPYCALL
 
         def bomb(*args):
             assert False, "shortcutting should have avoided this"
 
-        code.funcrun = bomb
-        code.funcrun_obj = bomb
+        fn.code.funcrun = bomb
+        fn.code.funcrun_obj = bomb
 
         w_3 = space.newint(3)
         w_res = space.call_function(fn, w_3)
@@ -756,18 +769,19 @@
     def test_flatcall_method(self):
         space = self.space
 
-        def f(self, a):
-            return a
-        code = PyCode._from_code(self.space, f.__code__)
-        fn = Function(self.space, code, self.space.newdict())
+        src = """
+def f(self, a):
+    return a
+"""
+        fn = self.compile(src)
 
         assert fn.code.fast_natural_arity == 2|PyCode.FLATPYCALL
 
         def bomb(*args):
             assert False, "shortcutting should have avoided this"
 
-        code.funcrun = bomb
-        code.funcrun_obj = bomb
+        fn.code.funcrun = bomb
+        fn.code.funcrun_obj = bomb
 
         w_3 = space.newint(3)
         w_res = space.appexec([fn, w_3], """(f, x):
@@ -784,9 +798,11 @@
     def test_flatcall_default_arg(self):
         space = self.space
 
-        def f(a, b):
-            return a+b
-        code = PyCode._from_code(self.space, f.__code__)
+        src = """
+def f(a, b):
+    return a+b
+"""
+        code = self.compile(src).code
         fn = Function(self.space, code, self.space.newdict(),
                       defs_w=[space.newint(1)])
 
@@ -813,9 +829,11 @@
     def test_flatcall_default_arg_method(self):
         space = self.space
 
-        def f(self, a, b):
-            return a+b
-        code = PyCode._from_code(self.space, f.__code__)
+        src = """
+def f(self, a, b):
+    return a+b
+        """
+        code = self.compile(src).code
         fn = Function(self.space, code, self.space.newdict(),
                       defs_w=[space.newint(1)])
 
diff --git a/pypy/interpreter/test/test_pycode.py b/pypy/interpreter/test/test_pycode.py
--- a/pypy/interpreter/test/test_pycode.py
+++ b/pypy/interpreter/test/test_pycode.py
@@ -14,6 +14,6 @@
     finally:
         sys.stdout = stdout
     print '>>>\n' + output + '\n<<<'
-    assert ' 1 (7)' in output
+    assert ' 0 (7)' in output
     assert ' 4 (None)' in output
-    assert ' 19 RETURN_VALUE ' in output
+    assert ' 16 RETURN_VALUE' in output
diff --git a/pypy/interpreter/test/test_zpy.py b/pypy/interpreter/test/test_zpy.py
--- a/pypy/interpreter/test/test_zpy.py
+++ b/pypy/interpreter/test/test_zpy.py
@@ -122,6 +122,3 @@
     # '5\n' --- this line sent to stderr
     assert ('\t<module>:           LOAD_NAME    0 (x)\n'
             '\t<module>:           PRINT_EXPR    0 \n') in output
-    assert ('\t<module>:           LOAD_CONST    0 (None)\n'
-            '\t<module>:           RETURN_VALUE    0 \n'
-            '>>>> ') in output
diff --git a/pypy/module/_ast/test/test_ast.py b/pypy/module/_ast/test/test_ast.py
--- a/pypy/module/_ast/test/test_ast.py
+++ b/pypy/module/_ast/test/test_ast.py
@@ -461,7 +461,7 @@
 
     def test_bug_null_in_objspace_type(self):
         import ast
-        code = ast.Expression(lineno=1, col_offset=1, body=ast.ListComp(lineno=1, col_offset=1, elt=ast.Call(lineno=1, col_offset=1, func=ast.Name(lineno=1, col_offset=1, id='str', ctx=ast.Load(lineno=1, col_offset=1)), args=[ast.Name(lineno=1, col_offset=1, id='x', ctx=ast.Load(lineno=1, col_offset=1))], keywords=[]), generators=[ast.comprehension(lineno=1, col_offset=1, target=ast.Name(lineno=1, col_offset=1, id='x', ctx=ast.Store(lineno=1, col_offset=1)), iter=ast.List(lineno=1, col_offset=1, elts=[ast.Num(lineno=1, col_offset=1, n=23)], ctx=ast.Load(lineno=1, col_offset=1, )), ifs=[])]))
+        code = ast.Expression(lineno=1, col_offset=1, body=ast.ListComp(lineno=1, col_offset=1, elt=ast.Call(lineno=1, col_offset=1, func=ast.Name(lineno=1, col_offset=1, id='str', ctx=ast.Load(lineno=1, col_offset=1)), args=[ast.Name(lineno=1, col_offset=1, id='x', ctx=ast.Load(lineno=1, col_offset=1))], keywords=[]), generators=[ast.comprehension(lineno=1, col_offset=1, target=ast.Name(lineno=1, col_offset=1, id='x', ctx=ast.Store(lineno=1, col_offset=1)), iter=ast.List(lineno=1, col_offset=1, elts=[ast.Num(lineno=1, col_offset=1, n=23)], ctx=ast.Load(lineno=1, col_offset=1, )), ifs=[], is_async=False)]))
         compile(code, '<template>', 'eval')
 
     def test_empty_yield_from(self):
diff --git a/pypy/module/imp/test/test_import.py b/pypy/module/imp/test/test_import.py
--- a/pypy/module/imp/test/test_import.py
+++ b/pypy/module/imp/test/test_import.py
@@ -788,7 +788,7 @@
     f.write(_getlong(mtime))
     if co:
         # marshal the code object with the PyPy marshal impl
-        pyco = PyCode._from_code(space, co)
+        pyco = space.createcompiler().compile(co, '?', 'exec', 0)
         w_marshal = space.getbuiltinmodule('marshal')
         w_marshaled_code = space.call_method(w_marshal, 'dumps', pyco)
         marshaled_code = space.bytes_w(w_marshaled_code)
@@ -809,7 +809,7 @@
     def test_read_compiled_module(self):
         space = self.space
         mtime = 12345
-        co = compile('x = 42', '?', 'exec')
+        co = 'x = 42'
         cpathname = _testfile(space, importing.get_pyc_magic(space), mtime, co)
         stream = streamio.open_file_as_stream(cpathname, "rb")
         try:
@@ -829,7 +829,7 @@
     def test_load_compiled_module(self):
         space = self.space
         mtime = 12345
-        co = compile('x = 42', '?', 'exec')
+        co = 'x = 42'
         cpathname = _testfile(space, importing.get_pyc_magic(space), mtime, co)
         w_modulename = space.wrap('somemodule')
         stream = streamio.open_file_as_stream(cpathname, "rb")
@@ -854,7 +854,7 @@
     def test_load_compiled_module_nopathname(self):
         space = self.space
         mtime = 12345
-        co = compile('x = 42', '?', 'exec')
+        co = 'x = 42'
         cpathname = _testfile(space, importing.get_pyc_magic(space), mtime, co)
         w_modulename = space.wrap('somemodule')
         stream = streamio.open_file_as_stream(cpathname, "rb")
@@ -931,7 +931,7 @@
                     continue
                 pathname = "whatever"
                 mtime = 12345
-                co = compile('x = 42', '?', 'exec')
+                co = 'x = 42'
                 cpathname = _testfile(space1, importing.get_pyc_magic(space1),
                                       mtime, co)
                 w_modulename = space2.wrap('somemodule')
diff --git a/pypy/tool/dis3.py b/pypy/tool/dis3.py
--- a/pypy/tool/dis3.py
+++ b/pypy/tool/dis3.py
@@ -1,20 +1,40 @@
-"""Disassembler of Python byte code into mnemonics.
-Python 3 dis.py partly backported to Python 2"""
+"""Disassembler of Python byte code into mnemonics."""
+
+from __future__ import print_function
 
 import sys
 import types
+import collections
+import io
 
 from opcode3 import *
 from opcode3 import __all__ as _opcodes_all
 
-__all__ = ["dis", "disassemble", "distb", "disco",
-           "findlinestarts", "findlabels"] + _opcodes_all
+__all__ = ["code_info", "dis", "disassemble", "distb", "disco",
+           "findlinestarts", "findlabels", "show_code",
+           "get_instructions", "Instruction", "Bytecode"] + _opcodes_all
 del _opcodes_all
 
-_have_code = (types.MethodType, types.FunctionType, types.CodeType, type)
+_have_code = (types.MethodType, types.FunctionType, types.CodeType,
+              classmethod, staticmethod, type)
+
+FORMAT_VALUE = opmap['FORMAT_VALUE']
+
+def _try_compile(source, name):
+    """Attempts to compile the given source, first as an expression and
+       then as a statement if the first approach fails.
+
+       Utility function to accept strings in functions that otherwise
+       expect code objects
+    """
+    try:
+        c = compile(source, name, 'eval')
+    except SyntaxError:
+        c = compile(source, name, 'exec')
+    return c
 
 def dis(x=None):
-    """Disassemble classes, methods, functions, or code.
+    """Disassemble classes, methods, functions, generators, or code.
 
     With no argument, disassemble the last traceback.
 
@@ -22,30 +42,31 @@
     if x is None:
         distb()
         return
-    if isinstance(x, types.InstanceType):
-        x = x.__class__
-    if hasattr(x, 'im_func'):
-        x = x.im_func
-    if hasattr(x, 'func_code'):
-        x = x.func_code
-    if hasattr(x, 'co_code'): # PyCode needs co_code before __dict__
+    if hasattr(x, '__func__'):  # Method
+        x = x.__func__
+    if hasattr(x, '__code__'):  # Function
+        x = x.__code__
+    if hasattr(x, 'gi_code'):  # Generator
+        x = x.gi_code
+    if hasattr(x, 'co_code'): # Code object
         disassemble(x)
-    elif hasattr(x, '__dict__'):
-        items = x.__dict__.items()
-        items.sort()
+    elif hasattr(x, '__dict__'):  # Class or module
+        items = sorted(x.__dict__.items())
         for name, x1 in items:
             if isinstance(x1, _have_code):
-                print "Disassembly of %s:" % name
+                print("Disassembly of %s:" % name)
                 try:
                     dis(x1)
                 except TypeError as msg:
-                    print "Sorry:", msg
-                print
-    elif isinstance(x, str):
-        disassemble_string(x)
+                    print("Sorry:", msg)
+                print()
+    elif isinstance(x, (bytes, bytearray)): # Raw bytecode
+        _disassemble_bytes(x)
+    elif isinstance(x, str):    # Source code
+        _disassemble_str(x)
     else:
-        raise TypeError("don't know how to disassemble %s objects" % \
-              type(x).__name__)
+        raise TypeError("don't know how to disassemble %s objects" %
+                        type(x).__name__)
 
 def distb(tb=None):
     """Disassemble a traceback (default: last traceback)."""
@@ -57,103 +78,290 @@
         while tb.tb_next: tb = tb.tb_next
     disassemble(tb.tb_frame.f_code, tb.tb_lasti)
 
+# The inspect module interrogates this dictionary to build its
+# list of CO_* constants. It is also used by pretty_flags to
+# turn the co_flags field into a human readable list.
+COMPILER_FLAG_NAMES = {
+     1: "OPTIMIZED",
+     2: "NEWLOCALS",
+     4: "VARARGS",
+     8: "VARKEYWORDS",
+    16: "NESTED",
+    32: "GENERATOR",
+    64: "NOFREE",
+   128: "COROUTINE",
+   256: "ITERABLE_COROUTINE",
+   512: "ASYNC_GENERATOR",
+}
+
+def pretty_flags(flags):
+    """Return pretty representation of code flags."""
+    names = []
+    for i in range(32):
+        flag = 1<<i
+        if flags & flag:
+            names.append(COMPILER_FLAG_NAMES.get(flag, hex(flag)))
+            flags ^= flag
+            if not flags:
+                break
+    else:
+        names.append(hex(flags))
+    return ", ".join(names)
+
+def _get_code_object(x):
+    """Helper to handle methods, functions, generators, strings and raw code objects"""
+    if hasattr(x, '__func__'): # Method
+        x = x.__func__
+    if hasattr(x, '__code__'): # Function
+        x = x.__code__
+    if hasattr(x, 'gi_code'):  # Generator
+        x = x.gi_code
+    if isinstance(x, str):     # Source code
+        x = _try_compile(x, "<disassembly>")
+    if hasattr(x, 'co_code'):  # Code object
+        return x
+    raise TypeError("don't know how to disassemble %s objects" %
+                    type(x).__name__)
+
+def code_info(x):
+    """Formatted details of methods, functions, or code."""
+    return _format_code_info(_get_code_object(x))
+
+def _format_code_info(co):
+    lines = []
+    lines.append("Name:              %s" % co.co_name)
+    lines.append("Filename:          %s" % co.co_filename)
+    lines.append("Argument count:    %s" % co.co_argcount)
+    lines.append("Kw-only arguments: %s" % co.co_kwonlyargcount)
+    lines.append("Number of locals:  %s" % co.co_nlocals)
+    lines.append("Stack size:        %s" % co.co_stacksize)
+    lines.append("Flags:             %s" % pretty_flags(co.co_flags))
+    if co.co_consts:
+        lines.append("Constants:")
+        for i_c in enumerate(co.co_consts):
+            lines.append("%4d: %r" % i_c)
+    if co.co_names:
+        lines.append("Names:")
+        for i_n in enumerate(co.co_names):
+            lines.append("%4d: %s" % i_n)
+    if co.co_varnames:
+        lines.append("Variable names:")
+        for i_n in enumerate(co.co_varnames):
+            lines.append("%4d: %s" % i_n)
+    if co.co_freevars:
+        lines.append("Free variables:")
+        for i_n in enumerate(co.co_freevars):
+            lines.append("%4d: %s" % i_n)
+    if co.co_cellvars:
+        lines.append("Cell variables:")
+        for i_n in enumerate(co.co_cellvars):
+            lines.append("%4d: %s" % i_n)
+    return "\n".join(lines)
+
+def show_code(co, file=None):
+    """Print details of methods, functions, or code to *file*.
+
+    If *file* is not provided, the output is printed on stdout.
+    """
+    print(code_info(co))
+
+_Instruction = collections.namedtuple("_Instruction",
+     "opname opcode arg argval argrepr offset starts_line is_jump_target")
+
+class Instruction(_Instruction):
+    """Details for a bytecode operation
+
+       Defined fields:
+         opname - human readable name for operation
+         opcode - numeric code for operation
+         arg - numeric argument to operation (if any), otherwise None
+         argval - resolved arg value (if known), otherwise same as arg
+         argrepr - human readable description of operation argument
+         offset - start index of operation within bytecode sequence
+         starts_line - line started by this opcode (if any), otherwise None
+         is_jump_target - True if other code jumps to here, otherwise False
+    """
+
+    def _disassemble(self, lineno_width=3, mark_as_current=False):
+        """Format instruction details for inclusion in disassembly output
+
+        *lineno_width* sets the width of the line number field (0 omits it)
+        *mark_as_current* inserts a '-->' marker arrow as part of the line
+        """
+        fields = []
+        # Column: Source code line number
+        if lineno_width:
+            if self.starts_line is not None:
+                lineno_fmt = "%%%dd" % lineno_width
+                fields.append(lineno_fmt % self.starts_line)
+            else:
+                fields.append(' ' * lineno_width)
+        # Column: Current instruction indicator
+        if mark_as_current:
+            fields.append('-->')
+        else:
+            fields.append('   ')
+        # Column: Jump target marker
+        if self.is_jump_target:
+            fields.append('>>')
+        else:
+            fields.append('  ')
+        # Column: Instruction offset from start of code sequence
+        fields.append(repr(self.offset).rjust(4))
+        # Column: Opcode name
+        fields.append(self.opname.ljust(20))
+        # Column: Opcode argument
+        if self.arg is not None:
+            fields.append(repr(self.arg).rjust(5))
+            # Column: Opcode argument details
+            if self.argrepr:
+                fields.append('(' + self.argrepr + ')')
+        return ' '.join(fields).rstrip()
+
+
+def get_instructions(x, first_line=None):
+    """Iterator for the opcodes in methods, functions or code
+
+    Generates a series of Instruction named tuples giving the details of
+    each operations in the supplied code.
+
+    If *first_line* is not None, it indicates the line number that should
+    be reported for the first source line in the disassembled code.
+    Otherwise, the source line information (if any) is taken directly from
+    the disassembled code object.
+    """
+    co = _get_code_object(x)
+    cell_names = co.co_cellvars + co.co_freevars
+    linestarts = dict(findlinestarts(co))
+    if first_line is not None:
+        line_offset = first_line - co.co_firstlineno
+    else:
+        line_offset = 0
+    return _get_instructions_bytes(co.co_code, co.co_varnames, co.co_names,
+                                   co.co_consts, cell_names, linestarts,
+                                   line_offset)
+
+def _get_const_info(const_index, const_list):
+    """Helper to get optional details about const references
+
+       Returns the dereferenced constant and its repr if the constant
+       list is defined.
+       Otherwise returns the constant index and its repr().
+    """
+    argval = const_index
+    if const_list is not None:
+        argval = const_list[const_index]
+    return argval, repr(argval)
+
+def _get_name_info(name_index, name_list):
+    """Helper to get optional details about named references
+
+       Returns the dereferenced name as both value and repr if the name
+       list is defined.
+       Otherwise returns the name index and its repr().
+    """
+    argval = name_index
+    if name_list is not None:
+        argval = name_list[name_index]
+        argrepr = argval
+    else:
+        argrepr = repr(argval)
+    return argval, argrepr
+
+
+def _get_instructions_bytes(code, varnames=None, names=None, constants=None,
+                      cells=None, linestarts=None, line_offset=0):
+    """Iterate over the instructions in a bytecode string.
+
+    Generates a sequence of Instruction namedtuples giving the details of each
+    opcode.  Additional information about the code's runtime environment
+    (e.g. variable names, constants) can be specified using optional
+    arguments.
+
+    """
+    labels = findlabels(code)
+    starts_line = None
+    for offset, op, arg in _unpack_opargs(code):
+        if linestarts is not None:
+            starts_line = linestarts.get(offset, None)
+            if starts_line is not None:
+                starts_line += line_offset
+        is_jump_target = offset in labels
+        argval = None
+        argrepr = ''
+        if arg is not None:
+            #  Set argval to the dereferenced value of the argument when
+            #  available, and argrepr to the string representation of argval.
+            #    _disassemble_bytes needs the string repr of the
+            #    raw name index for LOAD_GLOBAL, LOAD_CONST, etc.
+            argval = arg
+            if op in hasconst:
+                argval, argrepr = _get_const_info(arg, constants)
+            elif op in hasname:
+                argval, argrepr = _get_name_info(arg, names)
+            elif op in hasjrel:
+                argval = offset + 2 + arg
+                argrepr = "to " + repr(argval)
+            elif op in haslocal:
+                argval, argrepr = _get_name_info(arg, varnames)
+            elif op in hascompare:
+                argval = cmp_op[arg]
+                argrepr = argval
+            elif op in hasfree:
+                argval, argrepr = _get_name_info(arg, cells)
+            elif op == FORMAT_VALUE:
+                argval = ((None, str, repr, ascii)[arg & 0x3], bool(arg & 0x4))
+                argrepr = ('', 'str', 'repr', 'ascii')[arg & 0x3]
+                if argval[1]:
+                    if argrepr:
+                        argrepr += ', '
+                    argrepr += 'with format'
+        yield Instruction(opname[op], op,
+                          arg, argval, argrepr,
+                          offset, starts_line, is_jump_target)
+
 def disassemble(co, lasti=-1):
     """Disassemble a code object."""
-    code = co.co_code
-    labels = findlabels(code)
+    cell_names = co.co_cellvars + co.co_freevars
     linestarts = dict(findlinestarts(co))
-    n = len(code)
-    i = 0
-    extended_arg = 0
-    free = None
-    while i < n:
-        c = code[i]
-        op = ord(c)
-        if i in linestarts:
-            if i > 0:
-                print
-            print "%3d" % linestarts[i],
-        else:
-            print '   ',
+    _disassemble_bytes(co.co_code, lasti, co.co_varnames, co.co_names,
+                       co.co_consts, cell_names, linestarts)
 
-        if i == lasti: print '-->',
-        else: print '   ',
-        if i in labels: print '>>',
-        else: print '  ',
-        print repr(i).rjust(4),
-        print opname[op].ljust(20),
-        i = i+1
-        if op >= HAVE_ARGUMENT:
-            oparg = ord(code[i]) + ord(code[i+1])*256 + extended_arg
-            extended_arg = 0
-            i = i+2
-            if op == EXTENDED_ARG:
-                extended_arg = oparg*65536L
-            print repr(oparg).rjust(5),
-            if op in hasconst:
-                print '(' + repr(co.co_consts[oparg]) + ')',
-            elif op in hasname:
-                print '(' + co.co_names[oparg] + ')',
-            elif op in hasjrel:
-                print '(to ' + repr(i + oparg) + ')',
-            elif op in haslocal:
-                print '(' + co.co_varnames[oparg] + ')',
-            elif op in hascompare:
-                print '(' + cmp_op[oparg] + ')',
-            elif op in hasfree:
-                if free is None:
-                    free = co.co_cellvars + co.co_freevars
-                print '(' + free[oparg] + ')',
-            elif op in hasnargs:
-                print '(%d positional, %d keyword pair)' % \
-                      (ord(code[i-2]), ord(code[i-1])),
-        print
+def _disassemble_bytes(code, lasti=-1, varnames=None, names=None,
+                       constants=None, cells=None, linestarts=None,
+                       line_offset=0):
+    # Omit the line number column entirely if we have no line number info
+    show_lineno = linestarts is not None
+    # TODO?: Adjust width upwards if max(linestarts.values()) >= 1000?
+    lineno_width = 3 if show_lineno else 0
+    for instr in _get_instructions_bytes(code, varnames, names,
+                                         constants, cells, linestarts,
+                                         line_offset=line_offset):
+        new_source_line = (show_lineno and
+                           instr.starts_line is not None and
+                           instr.offset > 0)
+        if new_source_line:
+            print()
+        is_current_instr = instr.offset == lasti
+        print(instr._disassemble(lineno_width, is_current_instr))
 
-def disassemble_string(code, lasti=-1, varnames=None, names=None,
-                       constants=None):
-    labels = findlabels(code)
-    n = len(code)
-    i = 0
-    while i < n:
-        c = code[i]
-        op = ord(c)
-        if i == lasti: print '-->',
-        else: print '   ',
-        if i in labels: print '>>',
-        else: print '  ',
-        print repr(i).rjust(4),
-        print opname[op].ljust(15),
-        i = i+1
-        if op >= HAVE_ARGUMENT:
-            oparg = ord(code[i]) + ord(code[i+1])*256
-            i = i+2
-            print repr(oparg).rjust(5),
-            if op in hasconst:
-                if constants:
-                    print '(' + repr(constants[oparg]) + ')',
-                else:
-                    print '(%d)'%oparg,
-            elif op in hasname:
-                if names is not None:
-                    print '(' + names[oparg] + ')',
-                else:
-                    print '(%d)'%oparg,
-            elif op in hasjrel:
-                print '(to ' + repr(i + oparg) + ')',
-            elif op in haslocal:
-                if varnames:
-                    print '(' + varnames[oparg] + ')',
-                else:
-                    print '(%d)' % oparg,
-            elif op in hascompare:
-                print '(' + cmp_op[oparg] + ')',
-            elif op in hasnargs:
-                print '(%d positional, %d keyword pair)' % \
-                      (ord(code[i-2]), ord(code[i-1])),
-        print
+def _disassemble_str(source):
+    """Compile the source string, then disassemble the code object."""
+    disassemble(_try_compile(source, '<dis>'))
 
 disco = disassemble                     # XXX For backwards compatibility
 
+def _unpack_opargs(code):
+    extended_arg = 0
+    for i in range(0, len(code), 2):
+        op = ord(code[i])
+        if op >= HAVE_ARGUMENT:
+            arg = ord(code[i+1]) | extended_arg
+            extended_arg = (arg << 8) if op == EXTENDED_ARG else 0
+        else:
+            arg = None
+        yield (i, op, arg)
+
 def findlabels(code):
     """Detect all offsets in a byte code which are jump targets.
 
@@ -161,23 +369,16 @@
 
     """
     labels = []
-    n = len(code)
-    i = 0
-    while i < n:
-        c = code[i]
-        op = ord(c)
-        i = i+1
-        if op >= HAVE_ARGUMENT:
-            oparg = ord(code[i]) + ord(code[i+1])*256
-            i = i+2
-            label = -1
+    for offset, op, arg in _unpack_opargs(code):
+        if arg is not None:
             if op in hasjrel:
-                label = i+oparg
+                label = offset + 2 + arg
             elif op in hasjabs:
-                label = oparg
-            if label >= 0:
-                if label not in labels:
-                    labels.append(label)
+                label = arg
+            else:
+                continue
+            if label not in labels:
+                labels.append(label)
     return labels
 
 def findlinestarts(code):
@@ -186,13 +387,15 @@
     Generate pairs (offset, lineno) as described in Python/compile.c.
 
     """
-    byte_increments = [ord(c) for c in code.co_lnotab[0::2]]
-    line_increments = [ord(c) for c in code.co_lnotab[1::2]]
+    byte_increments = code.co_lnotab[0::2]
+    line_increments = code.co_lnotab[1::2]
 
     lastlineno = None
     lineno = code.co_firstlineno
     addr = 0
     for byte_incr, line_incr in zip(byte_increments, line_increments):
+        byte_incr = ord(byte_incr)
+        line_incr = ord(line_incr)
         if byte_incr:
             if lineno != lastlineno:
                 yield (addr, lineno)
@@ -205,27 +408,77 @@
     if lineno != lastlineno:
         yield (addr, lineno)
 
+class Bytecode:
+    """The bytecode operations of a piece of code
+
+    Instantiate this with a function, method, string of code, or a code object
+    (as returned by compile()).
+
+    Iterating over this yields the bytecode operations as Instruction instances.
+    """
+    def __init__(self, x, first_line=None, current_offset=None):
+        self.codeobj = co = _get_code_object(x)
+        if first_line is None:
+            self.first_line = co.co_firstlineno
+            self._line_offset = 0
+        else:
+            self.first_line = first_line
+            self._line_offset = first_line - co.co_firstlineno
+        self._cell_names = co.co_cellvars + co.co_freevars
+        self._linestarts = dict(findlinestarts(co))
+        self._original_object = x
+        self.current_offset = current_offset
+
+    def __iter__(self):
+        co = self.codeobj
+        return _get_instructions_bytes(co.co_code, co.co_varnames, co.co_names,
+                                       co.co_consts, self._cell_names,
+                                       self._linestarts,
+                                       line_offset=self._line_offset)
+
+    def __repr__(self):
+        return "{}({!r})".format(self.__class__.__name__,
+                                 self._original_object)
+
+    @classmethod
+    def from_traceback(cls, tb):
+        """ Construct a Bytecode from the given traceback """
+        while tb.tb_next:
+            tb = tb.tb_next
+        return cls(tb.tb_frame.f_code, current_offset=tb.tb_lasti)
+
+    def info(self):
+        """Return formatted information about the code object."""
+        return _format_code_info(self.codeobj)
+
+    def dis(self):
+        """Return a formatted view of the bytecode operations."""
+        co = self.codeobj
+        if self.current_offset is not None:
+            offset = self.current_offset
+        else:
+            offset = -1
+        with io.StringIO() as output:
+            _disassemble_bytes(co.co_code, varnames=co.co_varnames,
+                               names=co.co_names, constants=co.co_consts,
+                               cells=self._cell_names,
+                               linestarts=self._linestarts,
+                               line_offset=self._line_offset,
+                               file=output,
+                               lasti=offset)
+            return output.getvalue()
+
+
 def _test():
     """Simple test program to disassemble a file."""
-    if sys.argv[1:]:
-        if sys.argv[2:]:
-            sys.stderr.write("usage: python dis.py [-|file]\n")
-            sys.exit(2)
-        fn = sys.argv[1]
-        if not fn or fn == "-":
-            fn = None
-    else:
-        fn = None
-    if fn is None:
-        f = sys.stdin
-    else:
-        f = open(fn)
-    source = f.read()
-    if fn is not None:
-        f.close()
-    else:
-        fn = "<stdin>"
-    code = compile(source, fn, "exec")
+    import argparse
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument('infile', type=argparse.FileType(), nargs='?', default='-')
+    args = parser.parse_args()
+    with args.infile as infile:
+        source = infile.read()
+    code = compile(source, args.infile.name, "exec")
     dis(code)
 
 if __name__ == "__main__":
diff --git a/pypy/tool/opcode3.py b/pypy/tool/opcode3.py
--- a/pypy/tool/opcode3.py
+++ b/pypy/tool/opcode3.py
@@ -2,10 +2,8 @@
 """
 opcode module - potentially shared between dis and other modules which
 operate on bytecodes (e.g. peephole optimizers).
-"Backported" from Python 3 to Python 2 land - an excact copy of lib-python/3/opcode.py
 """
 
-
 __all__ = ["cmp_op", "hasconst", "hasname", "hasjrel", "hasjabs",
            "haslocal", "hascompare", "hasfree", "opname", "opmap",
            "HAVE_ARGUMENT", "EXTENDED_ARG", "hasnargs"]
@@ -33,10 +31,12 @@
 haslocal = []
 hascompare = []
 hasfree = []
-hasnargs = [] # unused
+hasnargs = []
 
 opmap = {}
-opname = ['<%r>' % (op,) for op in range(256)]
+opname = [''] * 256
+for op in range(256): opname[op] = '<%r>' % (op,)
+del op
 
 def def_op(name, op):
     opname[op] = name
@@ -174,9 +174,11 @@
 name_op('STORE_ANNOTATION', 127) # Index in name list
 
 def_op('RAISE_VARARGS', 130)    # Number of raise arguments (1, 2, or 3)
-def_op('CALL_FUNCTION', 131)    # #args
-def_op('MAKE_FUNCTION', 132)    # Flags
+def_op('CALL_FUNCTION', 131)    # #args + (#kwargs << 8)
+hasnargs.append(131)
+def_op('MAKE_FUNCTION', 132)    # Number of args with default values
 def_op('BUILD_SLICE', 133)      # Number of items
+def_op('MAKE_CLOSURE', 134)
 def_op('LOAD_CLOSURE', 135)
 hasfree.append(135)
 def_op('LOAD_DEREF', 136)
@@ -186,8 +188,12 @@
 def_op('DELETE_DEREF', 138)
 hasfree.append(138)
 
-def_op('CALL_FUNCTION_KW', 141)  # #args + #kwargs
-def_op('CALL_FUNCTION_EX', 142)  # Flags
+def_op('CALL_FUNCTION_VAR', 140)     # #args + (#kwargs << 8)
+hasnargs.append(140)
+def_op('CALL_FUNCTION_KW', 141)      # #args + (#kwargs << 8)
+hasnargs.append(141)
+def_op('CALL_FUNCTION_VAR_KW', 142)  # #args + (#kwargs << 8)
+hasnargs.append(142)
 
 jrel_op('SETUP_WITH', 143)
 
@@ -198,6 +204,8 @@
 def_op('LOAD_CLASSDEREF', 148)
 hasfree.append(148)
 
+jrel_op('SETUP_ASYNC_WITH', 154)
+
 def_op('EXTENDED_ARG', 144)
 EXTENDED_ARG = 144
 
@@ -207,12 +215,9 @@
 def_op('BUILD_TUPLE_UNPACK', 152)
 def_op('BUILD_SET_UNPACK', 153)
 
-jrel_op('SETUP_ASYNC_WITH', 154)
-
-def_op('FORMAT_VALUE', 155)
-def_op('BUILD_CONST_KEY_MAP', 156)
-def_op('BUILD_STRING', 157)
-def_op('BUILD_TUPLE_UNPACK_WITH_CALL', 158)
+def_op('FORMAT_VALUE', 155)   # in CPython 3.6, but available in PyPy from 3.5
+def_op("BUILD_CONST_KEY_MAP", 156)
+def_op('BUILD_STRING', 157)   # in CPython 3.6, but available in PyPy from 3.5
 
 # pypy modification, experimental bytecode
 def_op('LOOKUP_METHOD', 201)          # Index in name list
diff --git a/pypy/tool/pydis.py b/pypy/tool/pydis.py
deleted file mode 100644
--- a/pypy/tool/pydis.py
+++ /dev/null
@@ -1,202 +0,0 @@
-"""disassembler of Python byte code into mnemonics.
-
-XXX this only works for python-2.3 because of the linenumber 
-    optimization 
-
-"""
-
-import sys
-
-from pypy.tool import stdlib_opcode
-from pypy.tool.stdlib_opcode import *
-
-__all__ = ["dis","pydisassemble","distb","disco"] + stdlib_opcode.__all__
-
-EXTENDED_ARG = stdlib_opcode.opcodedesc.EXTENDED_ARG.index
-
-
-class Bytecode:
-    def __init__(self, disresult, bytecodeindex, oparg, lineno):
-        self.disresult = disresult 
-        self.index = bytecodeindex
-        self.op = ord(disresult.code.co_code[self.index])
-        self.name = opname[self.op]
-        self.oparg = oparg
-        self.lineno = lineno
-
-    def __eq__(self, other):
-        return (self.__class__ == other.__class__ and 
-                self.index == other.index and
-                self.op == other.op and
-                self.name == other.name and
-                self.oparg == other.oparg)
-
-    def __ne__(self, other):
-        return not (self == other)
-
-    def reprargstring(self, space = None):
-        """ return a string representation of any arguments. (empty for no args)"""
-        oparg = self.oparg
-        if oparg is None:
-            return ''
-        co = self.disresult.code
-        op = self.op
-        
-        s = repr(oparg).rjust(5) + " "
-        if op in hasconst:
-            consts = self.get_consts(space)
-            s += '(' + consts[oparg] + ')'
-        elif op in hasname:
-            s +=  '(' + co.co_names[oparg] + ')'
-        elif op in hasjrel:
-            s +=  '(to ' + repr(self.index + oparg) + ')'
-        elif op in haslocal:
-            s +=  '(' + co.co_varnames[oparg] + ')'
-        elif op in hascompare:
-            s +=  '(' + cmp_op[oparg] + ')'
-        elif op in hasfree:
-            #if free is None:
-            free = co.co_cellvars + co.co_freevars
-            s +=  '(' + free[oparg] + ')'
-        return s 
-
-    def get_consts(self, space=None):
-        # support both real code objects and PyCode objects
-        co = self.disresult.code
-        if hasattr(co, "co_consts"):
-            return [repr(c) for c in co.co_consts]
-
-        if space is None:
-            return [repr(c) for c in co.co_consts_w]
-        
-        r = lambda x: space.str_w(space.repr(x))
-        return [r(c) for c in co.co_consts_w]
-
-    def repr_with_space(self, space):
-        return self.name + self.reprargstring(space)
-
-    def __repr__(self):
-        return self.name + self.reprargstring()
-
-class DisResult:
-    """ an instance of this class gets returned for disassembling 
-        objects/functions/code objects whatever.    
-    """
-    def __init__(self, code):
-        self.code = code
-        self.bytecodes = []
-   
-    def append(self,  bytecodeindex, oparg, lineno):
-        """ append bytecode anaylsis information ..."""
-        bc = Bytecode(self, bytecodeindex, oparg, lineno)
-        self.bytecodes.append(bc)
-
-    def getbytecode(self, index):
-        """ return bytecode instance matching the given index. """
-        for bytecode in self.bytecodes:
-            if bytecode.index == index:
-                return bytecode
-        raise ValueError("no bytecode found on index %s in code \n%s" % (
-                index, pydis(self.code)))
-
-    def format(self):
-        lastlineno = -1
-        labels = findlabels(self.code.co_code)
-        lines = []
-        for bc in self.bytecodes:
-            l = []
-            if bc.lineno != lastlineno:
-                lastlineno = bc.lineno
-                l.append("%3d" % bc.lineno)
-            else:
-                l.append("   ")
-            l.append(bc.index in labels and ">>" or "  ")
-            l.append(repr(bc.index).rjust(4)) 
-            l.append(bc.name.ljust(20))
-            l.append(bc.reprargstring())
-            lines.append(" ".join(l))
-        return "\n".join(lines)
-
-    __repr__ = format
-
-def pydis(co): 
-    """return result of dissassembling a code object. """
-
-    if hasattr(co, 'func_code'):
-        co = co.func_code 
-
-    if hasattr(co, 'code'):
-        co = co.code 
-
-    disresult = DisResult(co)
-    code = co.co_code
-
-    byte_increments = [ord(c) for c in co.co_lnotab[0::2]]
-    line_increments = [ord(c) for c in co.co_lnotab[1::2]]
-    table_length = len(byte_increments)
-
-    lineno = co.co_firstlineno
-    table_index = 0
-    while (table_index < table_length
-           and byte_increments[table_index] == 0):
-        lineno += line_increments[table_index]
-        table_index += 1
-    addr = 0
-    line_incr = 0
-
-    n = len(code)
-    i = 0
-    extended_arg = 0
-    while i < n:
-        c = code[i]
-        op = ord(c)
-
-        if i >= addr:
-            lineno += line_incr
-            while table_index < table_length:
-                addr += byte_increments[table_index]
-                line_incr = line_increments[table_index]
-                table_index += 1
-                if line_incr:
-                    break
-            else:
-                addr = sys.maxint
-        current_bytecodeindex = i
-        i = i+1
-        oparg = None
-        if op >= HAVE_ARGUMENT:
-            oparg = ord(code[i]) + ord(code[i+1])*256 + extended_arg
-            extended_arg = 0
-            i = i+2
-            if op == EXTENDED_ARG:
-                extended_arg = oparg*65536L
-
-        disresult.append(current_bytecodeindex, oparg, lineno)
-    assert disresult is not None
-    return disresult
-
-def findlabels(code):
-    """Detect all offsets in a byte code which are jump targets.
-
-    Return the list of offsets.
-

From pypy.commits at gmail.com  Sat Jul 28 09:28:49 2018
From: pypy.commits at gmail.com (cfbolz)
Date: Sat, 28 Jul 2018 06:28:49 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: fix a test that
Message-ID: <5b5c6f91.1c69fb81.7d489.f5d6@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: py3.5
Changeset: r94919:09d5a78a60b4
Date: 2018-07-28 15:28 +0200
http://bitbucket.org/pypy/pypy/changeset/09d5a78a60b4/

Log:	fix a test that

diff --git a/pypy/interpreter/test/test_app_main.py b/pypy/interpreter/test/test_app_main.py
--- a/pypy/interpreter/test/test_app_main.py
+++ b/pypy/interpreter/test/test_app_main.py
@@ -470,7 +470,7 @@
 
     def test_cmd_co_name(self):
         child = self.spawn(['-c',
-                    'import sys; print sys._getframe(0).f_code.co_name'])
+                    'import sys; print(sys._getframe(0).f_code.co_name)'])
         child.expect('<module>')
 
     def test_ignore_python_inspect(self):

From pypy.commits at gmail.com  Sat Jul 28 10:04:19 2018
From: pypy.commits at gmail.com (cfbolz)
Date: Sat, 28 Jul 2018 07:04:19 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: fix test_executioncontext flakiness
Message-ID: <5b5c77e3.1c69fb81.50878.cd87@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: py3.5
Changeset: r94920:0370bd9179ff
Date: 2018-07-28 16:03 +0200
http://bitbucket.org/pypy/pypy/changeset/0370bd9179ff/

Log:	fix test_executioncontext flakiness

	when run in some combinations with other test files that set the
	file system encoding, space.appexec causes a call to "encode"
	internally, which produces extra C call events. Just ignore those.

diff --git a/pypy/interpreter/test/test_executioncontext.py b/pypy/interpreter/test/test_executioncontext.py
--- a/pypy/interpreter/test/test_executioncontext.py
+++ b/pypy/interpreter/test/test_executioncontext.py
@@ -149,7 +149,7 @@
         pass
         """)
         space.getexecutioncontext().setllprofile(None, None)
-        assert l == ['call', 'return', 'call', 'return']
+        assert l[-4:] == ['call', 'return', 'call', 'return']
 
     def test_llprofile_c_call(self):
         from pypy.interpreter.function import Function, Method
@@ -173,15 +173,15 @@
             return
             """ % snippet)
             space.getexecutioncontext().setllprofile(None, None)
-            assert l == ['call', 'return', 'call', 'c_call', 'c_return', 'return']
-            if isinstance(seen[0], Method):
-                w_class = space.type(seen[0].w_instance)
+            assert l[-6:] == ['call', 'return', 'call', 'c_call', 'c_return', 'return']
+            if isinstance(seen[-1], Method):
+                w_class = space.type(seen[-1].w_instance)
                 found = 'method %s of %s' % (
-                    seen[0].w_function.name,
+                    seen[-1].w_function.name,
                     w_class.getname(space).encode('utf-8'))
             else:
-                assert isinstance(seen[0], Function)
-                found = 'builtin %s' % seen[0].name
+                assert isinstance(seen[-1], Function)
+                found = 'builtin %s' % seen[-1].name
             assert found == expected_c_call
 
         check_snippet('l = []; l.append(42)', 'method append of list')
@@ -210,7 +210,7 @@
             return
             """ % snippet)
             space.getexecutioncontext().setllprofile(None, None)
-            assert l == ['call', 'return', 'call', 'c_call', 'c_exception', 'return']
+            assert l[-6:] == ['call', 'return', 'call', 'c_call', 'c_exception', 'return']
 
         check_snippet('d = {}; d.__getitem__(42)')
 

From pypy.commits at gmail.com  Sat Jul 28 15:32:06 2018
From: pypy.commits at gmail.com (cfbolz)
Date: Sat, 28 Jul 2018 12:32:06 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: fix the tests in test_pyframe that fail
 in combination with other files
Message-ID: <5b5cc4b6.1c69fb81.20157.c833@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: py3.5
Changeset: r94921:dfda715a5495
Date: 2018-07-28 21:31 +0200
http://bitbucket.org/pypy/pypy/changeset/dfda715a5495/

Log:	fix the tests in test_pyframe that fail in combination with other
	files

diff --git a/pypy/interpreter/test/test_pyframe.py b/pypy/interpreter/test/test_pyframe.py
--- a/pypy/interpreter/test/test_pyframe.py
+++ b/pypy/interpreter/test/test_pyframe.py
@@ -153,6 +153,8 @@
         r"""
         seen = []
         def tracer(f, event, *args):
+            if f.f_code.co_name == "decode":
+                return tracer
             seen.append((event, f.f_lineno))
             if len(seen) == 5:
                 f.f_lineno = 1       # bug shown only when setting lineno to 1
@@ -297,7 +299,8 @@
 
         l = []
         def trace(a,b,c):
-            l.append((a,b,c))
+            if a.f_code.co_name != "decode":
+                l.append((a,b,c))
 
         def f():
             h = _testing.Hidden()

From pypy.commits at gmail.com  Sat Jul 28 21:07:05 2018
From: pypy.commits at gmail.com (rlamy)
Date: Sat, 28 Jul 2018 18:07:05 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6: hg merge py3.5
Message-ID: <5b5d1339.1c69fb81.6bc64.e479@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.6
Changeset: r94922:326720df4fbe
Date: 2018-07-29 02:06 +0100
http://bitbucket.org/pypy/pypy/changeset/326720df4fbe/

Log:	hg merge py3.5

diff --git a/pypy/interpreter/astcompiler/codegen.py b/pypy/interpreter/astcompiler/codegen.py
--- a/pypy/interpreter/astcompiler/codegen.py
+++ b/pypy/interpreter/astcompiler/codegen.py
@@ -379,8 +379,8 @@
                                        names)
         self._visit_arg_annotations(args.kwonlyargs, names)
         kwarg = args.kwarg
-        if args.kwarg:
-            self._visit_arg_annotation(args.kwarg.arg, args.kwarg.annotation,
+        if kwarg:
+            self._visit_arg_annotation(kwarg.arg, kwarg.annotation,
                                        names)
         self._visit_arg_annotation("return", returns, names)
         l = len(names)
diff --git a/pypy/interpreter/astcompiler/symtable.py b/pypy/interpreter/astcompiler/symtable.py
--- a/pypy/interpreter/astcompiler/symtable.py
+++ b/pypy/interpreter/astcompiler/symtable.py
@@ -640,6 +640,10 @@
         assert isinstance(args, ast.arguments)
         if args.args:
             self._visit_arg_annotations(args.args)
+        if args.vararg:
+            self._visit_arg_annotation(args.vararg)
+        if args.kwarg:
+            self._visit_arg_annotation(args.kwarg)
         if args.kwonlyargs:
             self._visit_arg_annotations(args.kwonlyargs)
         if func.returns:
@@ -648,8 +652,11 @@
     def _visit_arg_annotations(self, args):
         for arg in args:
             assert isinstance(arg, ast.arg)
-            if arg.annotation:
-                arg.annotation.walkabout(self)
+            self._visit_arg_annotation(arg)
+
+    def _visit_arg_annotation(self, arg):
+        if arg.annotation:
+            arg.annotation.walkabout(self)
 
     def visit_Name(self, name):
         if name.ctx == ast.Load:
diff --git a/pypy/interpreter/test/test_app_main.py b/pypy/interpreter/test/test_app_main.py
--- a/pypy/interpreter/test/test_app_main.py
+++ b/pypy/interpreter/test/test_app_main.py
@@ -470,7 +470,7 @@
 
     def test_cmd_co_name(self):
         child = self.spawn(['-c',
-                    'import sys; print sys._getframe(0).f_code.co_name'])
+                    'import sys; print(sys._getframe(0).f_code.co_name)'])
         child.expect('<module>')
 
     def test_ignore_python_inspect(self):
diff --git a/pypy/interpreter/test/test_executioncontext.py b/pypy/interpreter/test/test_executioncontext.py
--- a/pypy/interpreter/test/test_executioncontext.py
+++ b/pypy/interpreter/test/test_executioncontext.py
@@ -149,7 +149,7 @@
         pass
         """)
         space.getexecutioncontext().setllprofile(None, None)
-        assert l == ['call', 'return', 'call', 'return']
+        assert l[-4:] == ['call', 'return', 'call', 'return']
 
     def test_llprofile_c_call(self):
         from pypy.interpreter.function import Function, Method
@@ -173,15 +173,15 @@
             return
             """ % snippet)
             space.getexecutioncontext().setllprofile(None, None)
-            assert l == ['call', 'return', 'call', 'c_call', 'c_return', 'return']
-            if isinstance(seen[0], Method):
-                w_class = space.type(seen[0].w_instance)
+            assert l[-6:] == ['call', 'return', 'call', 'c_call', 'c_return', 'return']
+            if isinstance(seen[-1], Method):
+                w_class = space.type(seen[-1].w_instance)
                 found = 'method %s of %s' % (
-                    seen[0].w_function.name,
+                    seen[-1].w_function.name,
                     w_class.getname(space).encode('utf-8'))
             else:
-                assert isinstance(seen[0], Function)
-                found = 'builtin %s' % seen[0].name
+                assert isinstance(seen[-1], Function)
+                found = 'builtin %s' % seen[-1].name
             assert found == expected_c_call
 
         check_snippet('l = []; l.append(42)', 'method append of list')
@@ -210,7 +210,7 @@
             return
             """ % snippet)
             space.getexecutioncontext().setllprofile(None, None)
-            assert l == ['call', 'return', 'call', 'c_call', 'c_exception', 'return']
+            assert l[-6:] == ['call', 'return', 'call', 'c_call', 'c_exception', 'return']
 
         check_snippet('d = {}; d.__getitem__(42)')
 
diff --git a/pypy/interpreter/test/test_pyframe.py b/pypy/interpreter/test/test_pyframe.py
--- a/pypy/interpreter/test/test_pyframe.py
+++ b/pypy/interpreter/test/test_pyframe.py
@@ -153,6 +153,8 @@
         r"""
         seen = []
         def tracer(f, event, *args):
+            if f.f_code.co_name == "decode":
+                return tracer
             seen.append((event, f.f_lineno))
             if len(seen) == 5:
                 f.f_lineno = 1       # bug shown only when setting lineno to 1
@@ -297,7 +299,8 @@
 
         l = []
         def trace(a,b,c):
-            l.append((a,b,c))
+            if a.f_code.co_name != "decode":
+                l.append((a,b,c))
 
         def f():
             h = _testing.Hidden()
diff --git a/pypy/interpreter/test/test_syntax.py b/pypy/interpreter/test/test_syntax.py
--- a/pypy/interpreter/test/test_syntax.py
+++ b/pypy/interpreter/test/test_syntax.py
@@ -689,6 +689,16 @@
                                     "bye" : 5, "kw" : 6, "return" : 42}
         """
 
+    def test_bug_annotations_lambda(self):
+        """
+        # those used to crash
+        def broken(*a: lambda x: None):
+            pass
+
+        def broken(**a: lambda x: None):
+            pass
+        """
+
 class AppTestSyntaxError:
 
     def test_tokenizer_error_location(self):

From pypy.commits at gmail.com  Sun Jul 29 04:06:43 2018
From: pypy.commits at gmail.com (wlav)
Date: Sun, 29 Jul 2018 01:06:43 -0700 (PDT)
Subject: [pypy-commit] pypy cppyy-packaging: fix typo
Message-ID: <5b5d7593.1c69fb81.bdc4d.62a8@mx.google.com>

Author: Wim Lavrijsen <WLavrijsen at lbl.gov>
Branch: cppyy-packaging
Changeset: r94923:3ef6ede5dc57
Date: 2018-07-28 17:48 -0700
http://bitbucket.org/pypy/pypy/changeset/3ef6ede5dc57/

Log:	fix typo

diff --git a/pypy/module/_cppyy/interp_cppyy.py b/pypy/module/_cppyy/interp_cppyy.py
--- a/pypy/module/_cppyy/interp_cppyy.py
+++ b/pypy/module/_cppyy/interp_cppyy.py
@@ -848,7 +848,7 @@
                 method = self.master.overloads[name]
             else:
             # try to match with run-time instantiations
-                # TODO: logically, this could be used, but in practice, it's proving to
+                # TODO: logically, this could be used, but in practice, it's proving too
                 #  greedy ... maybe as a last resort?
                 #for cppol in self.master.overloads.values():
                 #    try:

From pypy.commits at gmail.com  Sun Jul 29 04:06:46 2018
From: pypy.commits at gmail.com (wlav)
Date: Sun, 29 Jul 2018 01:06:46 -0700 (PDT)
Subject: [pypy-commit] pypy cppyy-packaging: (limited,
 through narrowing to double) support for long double
Message-ID: <5b5d7596.1c69fb81.8e36d.2d37@mx.google.com>

Author: Wim Lavrijsen <WLavrijsen at lbl.gov>
Branch: cppyy-packaging
Changeset: r94924:94c32f0372c1
Date: 2018-07-29 00:46 -0700
http://bitbucket.org/pypy/pypy/changeset/94c32f0372c1/

Log:	(limited, through narrowing to double) support for long double

diff --git a/pypy/module/_cppyy/capi/loadable_capi.py b/pypy/module/_cppyy/capi/loadable_capi.py
--- a/pypy/module/_cppyy/capi/loadable_capi.py
+++ b/pypy/module/_cppyy/capi/loadable_capi.py
@@ -28,10 +28,11 @@
 
 class _Arg:         # poor man's union
     _immutable_ = True
-    def __init__(self, tc, h = 0, l = -1, s = '', p = rffi.cast(rffi.VOIDP, 0)):
+    def __init__(self, tc, h = 0, l = -1, d = -1., s = '', p = rffi.cast(rffi.VOIDP, 0)):
         self.tc      = tc
         self._handle = h
         self._long   = l
+        self._double = d
         self._string = s
         self._voidp  = p
 
@@ -45,6 +46,11 @@
     def __init__(self, val):
         _Arg.__init__(self, 'l', l = val)
 
+class _ArgD(_Arg):
+    _immutable_ = True
+    def __init__(self, val):
+        _Arg.__init__(self, 'd', d = val)
+
 class _ArgS(_Arg):
     _immutable_ = True
     def __init__(self, val):
@@ -94,6 +100,9 @@
                     assert obj._voidp != rffi.cast(rffi.VOIDP, 0)
                     data = rffi.cast(rffi.VOIDPP, data)
                     data[0] = obj._voidp
+                elif obj.tc == 'd':
+                    assert isinstance(argtype, ctypeprim.W_CTypePrimitiveFloat)
+                    misc.write_raw_float_data(data, rffi.cast(rffi.DOUBLE, obj._double), argtype.size)
                 else:    # only other use is string
                     assert obj.tc == 's'
                     n = len(obj._string)
@@ -187,6 +196,7 @@
             'call_f'       : ([c_method, c_object, c_int, c_voidp],   c_float),
             'call_d'       : ([c_method, c_object, c_int, c_voidp],   c_double),
             'call_ld'      : ([c_method, c_object, c_int, c_voidp],   c_ldouble),
+            'call_nld'     : ([c_method, c_object, c_int, c_voidp],   c_double),
 
             'call_r'       : ([c_method, c_object, c_int, c_voidp],   c_voidp),
             # call_s actually takes an size_t* as last parameter, but this will do
@@ -410,7 +420,9 @@
     return rffi.cast(rffi.DOUBLE, space.float_w(call_capi(space, 'call_d', args)))
 def c_call_ld(space, cppmethod, cppobject, nargs, cargs):
     args = [_ArgH(cppmethod), _ArgH(cppobject), _ArgL(nargs), _ArgP(cargs)]
-    return rffi.cast(rffi.LONGDOUBLE, space.float_w(call_capi(space, 'call_ld', args)))
+    #return rffi.cast(rffi.LONGDOUBLE, space.float_w(call_capi(space, 'call_ld', args)))
+    # call_nld narrows long double to double
+    return rffi.cast(rffi.DOUBLE, space.float_w(call_capi(space, 'call_nld', args)))
 
 def c_call_r(space, cppmethod, cppobject, nargs, cargs):
     args = [_ArgH(cppmethod), _ArgH(cppobject), _ArgL(nargs), _ArgP(cargs)]
@@ -664,6 +676,11 @@
 def c_stdstring2stdstring(space, cppobject):
     return _cdata_to_cobject(space, call_capi(space, 'stdstring2stdstring', [_ArgH(cppobject)]))
 
+def c_longdouble2double(space, addr):
+    return space.float_w(call_capi(space, 'longdouble2double', [_ArgP(addr)]))
+def c_double2longdouble(space, dval, addr):
+    call_capi(space, 'double2longdouble', [_ArgD(dval), _ArgP(addr)])
+
 def c_vectorbool_getitem(space, vbool, idx):
     return call_capi(space, 'vectorbool_getitem', [_ArgH(vbool), _ArgL(idx)])
 def c_vectorbool_setitem(space, vbool, idx, value):
diff --git a/pypy/module/_cppyy/converter.py b/pypy/module/_cppyy/converter.py
--- a/pypy/module/_cppyy/converter.py
+++ b/pypy/module/_cppyy/converter.py
@@ -211,6 +211,9 @@
         x[0] = self._unwrap_object(space, w_obj)
 
     def default_argument_libffi(self, space, address):
+        if not self.valid_default:
+            from pypy.module._cppyy.interp_cppyy import FastCallNotPossible
+            raise FastCallNotPossible
         x = rffi.cast(self.c_ptrtype, address)
         x[0] = self.default
 
@@ -224,7 +227,7 @@
         rffiptr = rffi.cast(self.c_ptrtype, address)
         rffiptr[0] = self._unwrap_object(space, w_value)
 
-class ConstRefNumericTypeConverterMixin(NumericTypeConverterMixin):
+class ConstRefNumericTypeConverterMixin(object):
     _mixin_ = True
 
     def cffi_type(self, space):
@@ -321,59 +324,92 @@
     pass
 
 class FloatConverter(ffitypes.typeid(rffi.FLOAT), FloatTypeConverterMixin, TypeConverter):
-    _immutable_fields_ = ['default']
+    _immutable_fields_ = ['default', 'valid_default']
 
     def __init__(self, space, default):
-        if default:
+        self.valid_default = False
+        try:
             fval = float(rfloat.rstring_to_float(default))
-        else:
+            self.valid_default = True
+        except Exception:
             fval = float(0.)
-        self.default = r_singlefloat(fval)
+        self.default = rffi.cast(rffi.FLOAT, r_singlefloat(fval))
 
     def from_memory(self, space, w_obj, offset):
         address = self._get_raw_address(space, w_obj, offset)
         rffiptr = rffi.cast(self.c_ptrtype, address)
         return self._wrap_object(space, rffiptr[0])
 
-class ConstFloatRefConverter(FloatConverter):
+class ConstFloatRefConverter(ConstRefNumericTypeConverterMixin, FloatConverter):
     _immutable_fields_ = ['typecode']
     typecode = 'f'
 
-    def cffi_type(self, space):
-        state = space.fromcache(ffitypes.State)
-        return state.c_voidp
-
-    def convert_argument_libffi(self, space, w_obj, address, scratch):
-        from pypy.module._cppyy.interp_cppyy import FastCallNotPossible
-        raise FastCallNotPossible
-
 class DoubleConverter(ffitypes.typeid(rffi.DOUBLE), FloatTypeConverterMixin, TypeConverter):
-    _immutable_fields_ = ['default']
+    _immutable_fields_ = ['default', 'valid_default']
 
     def __init__(self, space, default):
-        if default:
+        self.valid_default = False
+        try:
             self.default = rffi.cast(self.c_type, rfloat.rstring_to_float(default))
-        else:
+            self.valid_default = True
+        except Exception:
             self.default = rffi.cast(self.c_type, 0.)
 
 class ConstDoubleRefConverter(ConstRefNumericTypeConverterMixin, DoubleConverter):
     _immutable_fields_ = ['typecode']
     typecode = 'd'
 
-class LongDoubleConverter(ffitypes.typeid(rffi.LONGDOUBLE), FloatTypeConverterMixin, TypeConverter):
-    _immutable_fields_ = ['default']
+class LongDoubleConverter(TypeConverter):
+    _immutable_fields_ = ['default', 'valid_default']
+    typecode = 'g'
 
     def __init__(self, space, default):
-        if default:
-            fval = float(rfloat.rstring_to_float(default))
-        else:
-            fval = float(0.)
-        self.default = r_longfloat(fval)
+        self.valid_default = False
+        try:
+            # use float() instead of cast with r_longfloat
+            fval = rffi.cast(rffi.DOUBLE, rfloat.rstring_to_float(default))
+            self.valid_default = True
+        except Exception:
+            fval = rffi.cast(rffi.DOUBLE, 0.)
+        #self.default = r_longfloat(fval)
+        self.default = fval
+
+    def convert_argument(self, space, w_obj, address):
+        x = rffi.cast(rffi.VOIDP, address)
+        capi.c_double2longdouble(space, space.float_w(w_obj), x)
+        ba = rffi.cast(rffi.CCHARP, address)
+        ba[capi.c_function_arg_typeoffset(space)] = self.typecode
+
+    def convert_argument_libffi(self, space, w_obj, address, scratch):
+        x = rffi.cast(rffi.VOIDP, address)
+        capi.c_double2longdouble(space, space.float_w(w_obj), x)
+
+    def default_argument_libffi(self, space, address):
+        if not self.valid_default:
+            from pypy.module._cppyy.interp_cppyy import FastCallNotPossible
+            raise FastCallNotPossible
+        x = rffi.cast(rffi.VOIDP, address)
+        capi.c_double2longdouble(space, self.default, x)
+
+    def from_memory(self, space, w_obj, offset):
+        address = self._get_raw_address(space, w_obj, offset)
+        rffiptr = rffi.cast(rffi.VOIDP, address)
+        return space.newfloat(capi.c_longdouble2double(space, rffiptr))
+
+    def to_memory(self, space, w_obj, w_value, offset):
+        address = self._get_raw_address(space, w_obj, offset)
+        rffiptr = rffi.cast(rffi.VOIDP, address)
+        capi.c_double2longdouble(space, space.float_w(w_value), rffiptr)
 
 class ConstLongDoubleRefConverter(ConstRefNumericTypeConverterMixin, LongDoubleConverter):
     _immutable_fields_ = ['typecode']
     typecode = 'g'
 
+    def convert_argument_libffi(self, space, w_obj, address, scratch):
+        capi.c_double2longdouble(space, space.float_w(w_obj), rffi.cast(rffi.VOIDP, scratch))
+        x = rffi.cast(rffi.VOIDPP, address)
+        x[0] = scratch
+
 
 class CStringConverter(TypeConverter):
     def convert_argument(self, space, w_obj, address):
@@ -949,8 +985,8 @@
 _converters["const float&"]             = ConstFloatRefConverter
 _converters["double"]                   = DoubleConverter
 _converters["const double&"]            = ConstDoubleRefConverter
-#_converters["long double"]              = LongDoubleConverter
-#_converters["const long double&"]       = ConstLongDoubleRefConverter
+_converters["long double"]              = LongDoubleConverter
+_converters["const long double&"]       = ConstLongDoubleRefConverter
 _converters["const char*"]              = CStringConverter
 _converters["void*"]                    = VoidPtrConverter
 _converters["void**"]                   = VoidPtrPtrConverter
@@ -984,7 +1020,12 @@
             _immutable_ = True
             typecode = c_tc
             def __init__(self, space, default):
-                self.default = rffi.cast(self.c_type, capi.c_strtoll(space, default))
+                self.valid_default = False
+                try:
+                    self.default = rffi.cast(self.c_type, capi.c_strtoll(space, default))
+                    self.valid_default = True
+                except Exception:
+                    self.default = rffi.cast(self.c_type, 0)
         class ConstRefConverter(ConstRefNumericTypeConverterMixin, BasicConverter):
             _immutable_ = True
         for name in names:
@@ -1001,7 +1042,12 @@
             _immutable_ = True
             typecode = c_tc
             def __init__(self, space, default):
-                self.default = rffi.cast(self.c_type, capi.c_strtoll(space, default))
+                self.valid_default = False
+                try:
+                    self.default = rffi.cast(self.c_type, capi.c_strtoll(space, default))
+                    self.valid_default = True
+                except Exception:
+                    self.default = rffi.cast(self.c_type, 0)
         class ConstRefConverter(ConstRefNumericTypeConverterMixin, BasicConverter):
             _immutable_ = True
         for name in names:
@@ -1021,7 +1067,12 @@
             _immutable_ = True
             typecode = c_tc
             def __init__(self, space, default):
-                self.default = rffi.cast(self.c_type, capi.c_strtoull(space, default))
+                self.valid_default = False
+                try:
+                    self.default = rffi.cast(self.c_type, capi.c_strtoull(space, default))
+                    self.valid_default = True
+                except Exception:
+                    self.default = rffi.cast(self.c_type, 0)
         class ConstRefConverter(ConstRefNumericTypeConverterMixin, BasicConverter):
             _immutable_ = True
         for name in names:
diff --git a/pypy/module/_cppyy/executor.py b/pypy/module/_cppyy/executor.py
--- a/pypy/module/_cppyy/executor.py
+++ b/pypy/module/_cppyy/executor.py
@@ -76,9 +76,6 @@
 class NumericExecutorMixin(object):
     _mixin_ = True
 
-    #def _wrap_object(self, space, obj):
-    #    return getattr(space, self.wrapper)(obj)
-
     def execute(self, space, cppmethod, cppthis, num_args, args):
         result = self.c_stubcall(space, cppmethod, cppthis, num_args, args)
         return self._wrap_object(space, rffi.cast(self.c_type, result))
@@ -100,13 +97,10 @@
         self.w_item = w_item
         self.do_assign = True
 
-    #def _wrap_object(self, space, obj):
-    #    return getattr(space, self.wrapper)(rffi.cast(self.c_type, obj))
-
     def _wrap_reference(self, space, rffiptr):
         if self.do_assign:
             rffiptr[0] = rffi.cast(self.c_type, self._unwrap_object(space, self.w_item))
-        self.do_assign = False
+            self.do_assign = False
         return self._wrap_object(space, rffiptr[0])    # all paths, for rtyper
 
     def execute(self, space, cppmethod, cppthis, num_args, args):
@@ -119,6 +113,48 @@
         return self._wrap_reference(space,
             rffi.cast(self.c_ptrtype, rffi.cast(rffi.VOIDPP, result)[0]))
 
+class LongDoubleExecutorMixin(object):
+    # Note: not really supported, but returns normal double
+    _mixin_ = True
+
+    def execute(self, space, cppmethod, cppthis, num_args, args):
+        result = self.c_stubcall(space, cppmethod, cppthis, num_args, args)
+        return space.newfloat(result)
+
+    def execute_libffi(self, space, cif_descr, funcaddr, buffer):
+        from pypy.module._cppyy.interp_cppyy import FastCallNotPossible
+        raise FastCallNotPossible
+
+class LongDoubleExecutor(ffitypes.typeid(rffi.LONGDOUBLE), LongDoubleExecutorMixin, Executor):
+    _immutable_ = True
+    c_stubcall  = staticmethod(capi.c_call_ld)
+
+class LongDoubleRefExecutorMixin(NumericRefExecutorMixin):
+    # Note: not really supported, but returns normal double
+    _mixin_ = True
+
+    def _wrap_reference(self, space, rffiptr):
+        if self.do_assign:
+            capi.c_double2longdouble(space, space.float_w(self.w_item), rffiptr)
+            self.do_assign = False
+            return self.w_item
+        return space.newfloat(capi.c_longdouble2double(space, rffiptr))
+
+    def execute(self, space, cppmethod, cppthis, num_args, args):
+        result = capi.c_call_r(space, cppmethod, cppthis, num_args, args)
+        return self._wrap_reference(space, rffi.cast(self.c_ptrtype, result))
+
+    def execute_libffi(self, space, cif_descr, funcaddr, buffer):
+        jit_libffi.jit_ffi_call(cif_descr, funcaddr, buffer)
+        result = rffi.ptradd(buffer, cif_descr.exchange_result)
+        return self._wrap_reference(space,
+            rffi.cast(self.c_ptrtype, rffi.cast(rffi.VOIDPP, result)[0]))
+
+class LongDoubleRefExecutor(ffitypes.typeid(rffi.LONGDOUBLE), LongDoubleRefExecutorMixin, Executor):
+    def cffi_type(self, space):
+        state = space.fromcache(ffitypes.State)
+        return state.c_voidp
+
 
 class CStringExecutor(Executor):
     def execute(self, space, cppmethod, cppthis, num_args, args):
@@ -341,6 +377,10 @@
 _executors["void*"]               = PtrTypeExecutor
 _executors["const char*"]         = CStringExecutor
 
+# long double not really supported: narrows to double
+_executors["long double"]          = LongDoubleExecutor
+_executors["long double&"]         = LongDoubleRefExecutor
+
 # special cases (note: 'string' aliases added below)
 _executors["constructor"]         = ConstructorExecutor
 
diff --git a/pypy/module/_cppyy/ffitypes.py b/pypy/module/_cppyy/ffitypes.py
--- a/pypy/module/_cppyy/ffitypes.py
+++ b/pypy/module/_cppyy/ffitypes.py
@@ -296,7 +296,8 @@
     _immutable_fields_ = ['c_type', 'c_ptrtype', 'typecode']
 
     c_type      = rffi.LONGDOUBLE
-    c_ptrtype   = rffi.LONGDOUBLEP
+    # c_ptrtype   = rffi.LONGDOUBLEP   # useless type at this point
+    c_ptrtype   = rffi.VOIDP
     typecode    = 'g'
 
     # long double is not really supported ...
@@ -304,7 +305,7 @@
         return r_longfloat(space.float_w(w_obj))
 
     def _wrap_object(self, space, obj):
-        return space.wrap(obj)
+        return space.newfloat(obj)
 
     def cffi_type(self, space):
         state = space.fromcache(State)
diff --git a/pypy/module/_cppyy/include/capi.h b/pypy/module/_cppyy/include/capi.h
--- a/pypy/module/_cppyy/include/capi.h
+++ b/pypy/module/_cppyy/include/capi.h
@@ -63,6 +63,8 @@
     double cppyy_call_d(cppyy_method_t method, cppyy_object_t self, int nargs, void* args);
     RPY_EXTERN
     long double cppyy_call_ld(cppyy_method_t method, cppyy_object_t self, int nargs, void* args);
+    RPY_EXTERN
+    double cppyy_call_nld(cppyy_method_t method, cppyy_object_t self, int nargs, void* args);
 
     RPY_EXTERN
     void* cppyy_call_r(cppyy_method_t method, cppyy_object_t self, int nargs, void* args);
@@ -220,6 +222,11 @@
     cppyy_object_t cppyy_stdstring2stdstring(cppyy_object_t ptr);
 
     RPY_EXTERN
+    double cppyy_longdouble2double(void*);
+    RPY_EXTERN
+    void   cppyy_double2longdouble(double, void*);
+
+    RPY_EXTERN
     int         cppyy_vectorbool_getitem(cppyy_object_t ptr, int idx);
     RPY_EXTERN
     void        cppyy_vectorbool_setitem(cppyy_object_t ptr, int idx, int value);
diff --git a/pypy/module/_cppyy/interp_cppyy.py b/pypy/module/_cppyy/interp_cppyy.py
--- a/pypy/module/_cppyy/interp_cppyy.py
+++ b/pypy/module/_cppyy/interp_cppyy.py
@@ -230,8 +230,10 @@
         if self.converters is None:
             try:
                 self._setup(cppthis)
-            except Exception:
-                pass
+            except Exception as e:
+                if self.converters is None:
+                    raise oefmt(self.space.w_SystemError,
+                        "unable to initialize converters (%s)", str(e))
 
         # attempt to call directly through ffi chain
         if useffi and self._funcaddr:
diff --git a/pypy/module/_cppyy/test/datatypes.cxx b/pypy/module/_cppyy/test/datatypes.cxx
--- a/pypy/module/_cppyy/test/datatypes.cxx
+++ b/pypy/module/_cppyy/test/datatypes.cxx
@@ -113,6 +113,7 @@
 float                CppyyTestData::get_float()   { return m_float; }
 double               CppyyTestData::get_double()  { return m_double; }
 long double          CppyyTestData::get_ldouble() { return m_ldouble; }
+long double          CppyyTestData::get_ldouble_def(long double ld) { return ld; }
 CppyyTestData::EWhat CppyyTestData::get_enum()    { return m_enum; }
 void*                CppyyTestData::get_voidp()   { return m_voidp; }
 
diff --git a/pypy/module/_cppyy/test/datatypes.h b/pypy/module/_cppyy/test/datatypes.h
--- a/pypy/module/_cppyy/test/datatypes.h
+++ b/pypy/module/_cppyy/test/datatypes.h
@@ -96,6 +96,8 @@
     float                get_float();
     double               get_double();
     long double          get_ldouble();
+    typedef long double aap_t;
+    long double          get_ldouble_def(long double ld = aap_t(1));
     EWhat                get_enum();
     void*                get_voidp();
 
diff --git a/pypy/module/_cppyy/test/test_datatypes.py b/pypy/module/_cppyy/test/test_datatypes.py
--- a/pypy/module/_cppyy/test/test_datatypes.py
+++ b/pypy/module/_cppyy/test/test_datatypes.py
@@ -56,10 +56,11 @@
         assert round(c.m_double         + 77., 11) == 0
         assert round(c.get_double_cr()  + 77., 11) == 0
         assert round(c.get_double_r()   + 77., 11) == 0
-        #assert round(c.m_ldouble        + 88., 24) == 0
-        #assert round(c.get_ldouble_cr() + 88., 24) == 0
-        #assert round(c.get_ldouble_r()  + 88., 24) == 0
-        assert round(c.m_double + 77., 8) == 0
+        assert round(c.m_ldouble        + 88., 24) == 0
+        assert round(c.get_ldouble_cr() + 88., 24) == 0
+        assert round(c.get_ldouble_r()  + 88., 24) == 0
+        assert round(c.get_ldouble_def()  -1., 24) == 0
+        assert round(c.get_ldouble_def(2) -2., 24) == 0
 
         """# complex<double> type
         assert type(c.get_complex()) == complex
@@ -187,16 +188,20 @@
             assert eval('c.m_%s' % names[i]) == 3*i
 
         # float types through functions
-        c.set_float( 0.123 );  assert round(c.get_float()  - 0.123, 5) == 0
-        c.set_double( 0.456 ); assert round(c.get_double() - 0.456, 8) == 0
+        c.set_float(0.123);   assert round(c.get_float()   - 0.123, 5) == 0
+        c.set_double(0.456);  assert round(c.get_double()  - 0.456, 8) == 0
+        c.set_ldouble(0.789); assert round(c.get_ldouble() - 0.789, 8) == 0
 
         # float types through data members
-        c.m_float = 0.123;      assert round(c.get_float()  - 0.123, 5) == 0
-        c.set_float(0.234);     assert round(c.m_float      - 0.234, 5) == 0
-        c.set_float_cr(0.456);  assert round(c.m_float      - 0.456, 5) == 0
-        c.m_double = 0.678;     assert round(c.get_double() - 0.678, 8) == 0
-        c.set_double(0.890);    assert round(c.m_double     - 0.890, 8) == 0
-        c.set_double_cr(0.012); assert round(c.m_double     - 0.012, 8) == 0
+        c.m_float = 0.123;       assert round(c.get_float()   - 0.123, 5) == 0
+        c.set_float(0.234);      assert round(c.m_float       - 0.234, 5) == 0
+        c.set_float_cr(0.456);   assert round(c.m_float       - 0.456, 5) == 0
+        c.m_double = 0.678;      assert round(c.get_double()  - 0.678, 8) == 0
+        c.set_double(0.890);     assert round(c.m_double      - 0.890, 8) == 0
+        c.set_double_cr(0.012);  assert round(c.m_double      - 0.012, 8) == 0
+        c.m_ldouble = 0.876;     assert round(c.get_ldouble() - 0.876, 8) == 0
+        c.set_ldouble(0.098);    assert round(c.m_ldouble     - 0.098, 8) == 0
+        c.set_ldouble_cr(0.210); assert round(c.m_ldouble     - 0.210, 8) == 0
 
         # arrays; there will be pointer copies, so destroy the current ones
         c.destroy_arrays()
@@ -295,10 +300,12 @@
         assert CppyyTestData.s_ullong   ==  404
 
         # floating point types
-        assert round(CppyyTestData.s_float  + 606., 5) == 0
-        assert round(c.s_float              + 606., 5) == 0
-        assert round(CppyyTestData.s_double + 707., 8) == 0
-        assert round(c.s_double             + 707., 8) == 0
+        assert round(CppyyTestData.s_float   + 606., 5) == 0
+        assert round(c.s_float               + 606., 5) == 0
+        assert round(CppyyTestData.s_double  + 707., 8) == 0
+        assert round(c.s_double              + 707., 8) == 0
+        assert round(CppyyTestData.s_ldouble + 808., 8) == 0
+        assert round(c.s_ldouble             + 808., 8) == 0
 
         c.__destruct__()
 
@@ -363,6 +370,10 @@
         assert CppyyTestData.s_double             == -math.pi
         CppyyTestData.s_double                     =  math.pi
         assert c.s_double                         ==  math.pi
+        c.s_ldouble                                = -math.pi
+        assert CppyyTestData.s_ldouble            == -math.pi
+        CppyyTestData.s_ldouble                    =  math.pi
+        assert c.s_ldouble                        ==  math.pi
 
         c.__destruct__()
 

From pypy.commits at gmail.com  Sun Jul 29 05:51:33 2018
From: pypy.commits at gmail.com (rlamy)
Date: Sun, 29 Jul 2018 02:51:33 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6: Follow-up on 56432e2: fix .format() test
 as well
Message-ID: <5b5d8e25.1c69fb81.6fd7.baca@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.6
Changeset: r94925:42970c94abc5
Date: 2018-07-29 10:50 +0100
http://bitbucket.org/pypy/pypy/changeset/42970c94abc5/

Log:	Follow-up on 56432e2: fix .format() test as well

diff --git a/pypy/objspace/std/test/test_newformat.py b/pypy/objspace/std/test/test_newformat.py
--- a/pypy/objspace/std/test/test_newformat.py
+++ b/pypy/objspace/std/test/test_newformat.py
@@ -486,4 +486,5 @@
             pass
 
         excinfo = raises(ValueError, "{:j}".format, x(1))
-        assert str(excinfo.value) == "unknown format code j for object of type 'x'"
+        print(excinfo.value)
+        assert str(excinfo.value) == "Unknown format code j for object of type 'x'"

From pypy.commits at gmail.com  Sun Jul 29 06:59:55 2018
From: pypy.commits at gmail.com (rlamy)
Date: Sun, 29 Jul 2018 03:59:55 -0700 (PDT)
Subject: [pypy-commit] pypy py3tests: hg merge py3.5
Message-ID: <5b5d9e2b.1c69fb81.bfd52.a3b1@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3tests
Changeset: r94926:b6b0feb7e7f0
Date: 2018-07-29 11:58 +0100
http://bitbucket.org/pypy/pypy/changeset/b6b0feb7e7f0/

Log:	hg merge py3.5

diff too long, truncating to 2000 out of 25451 lines

diff --git a/.hgtags b/.hgtags
--- a/.hgtags
+++ b/.hgtags
@@ -33,7 +33,12 @@
 050d84dd78997f021acf0e133934275d63547cc0 release-pypy2.7-v5.4.1
 050d84dd78997f021acf0e133934275d63547cc0 release-pypy2.7-v5.4.1
 0e2d9a73f5a1818d0245d75daccdbe21b2d5c3ef release-pypy2.7-v5.4.1
+4909c06daf41ce88f87dc01c57959cadad4df4a8 RevDB-pypy2.7-v5.4.1
+4909c06daf41ce88f87dc01c57959cadad4df4a8 RevDB-pypy2.7-v5.4.1
+d7724c0a5700b895a47de44074cdf5fd659a988f RevDB-pypy2.7-v5.4.1
 aff251e543859ce4508159dd9f1a82a2f553de00 release-pypy2.7-v5.6.0
+e90317857d27917bf840caf675832292ee070510 RevDB-pypy2.7-v5.6.1
+a24d6c7000c8099c73d3660857f7e3cee5ac045c RevDB-pypy2.7-v5.6.2
 fa3249d55d15b9829e1be69cdf45b5a44cec902d release-pypy2.7-v5.7.0
 b16a4363e930f6401bceb499b9520955504c6cb0 release-pypy3.5-v5.7.0
 1aa2d8e03cdfab54b7121e93fda7e98ea88a30bf release-pypy2.7-v5.7.1
@@ -51,3 +56,5 @@
 0000000000000000000000000000000000000000 release-pypy3.5-v5.10.0
 09f9160b643e3f02ccb8c843b2fbb4e5cbf54082 release-pypy3.5-v5.10.0
 3f6eaa010fce78cc7973bdc1dfdb95970f08fed2 release-pypy3.5-v5.10.1
+ab0b9caf307db6592905a80b8faffd69b39005b8 release-pypy2.7-v6.0.0
+fdd60ed87e941677e8ea11acf9f1819466521bf2 release-pypy3.5-v6.0.0
diff --git a/LICENSE b/LICENSE
--- a/LICENSE
+++ b/LICENSE
@@ -247,6 +247,7 @@
   Lukas Vacek
   Omer Katz
   Jacek Generowicz
+  Tomasz Dziopa
   Sylvain Thenault
   Jakub Stasiak
   Andrew Dalke
@@ -307,6 +308,7 @@
   Yury V. Zaytsev
   florinpapa
   Anders Sigfridsson
+  Matt Jackson
   Nikolay Zinov
   rafalgalczynski at gmail.com
   Joshua Gilbert
diff --git a/dotviewer/font/NOTICE b/dotviewer/font/COPYING.txt
rename from dotviewer/font/NOTICE
rename to dotviewer/font/COPYING.txt
diff --git a/lib-python/3/datetime.py b/lib-python/3/datetime.py
--- a/lib-python/3/datetime.py
+++ b/lib-python/3/datetime.py
@@ -7,6 +7,9 @@
 import time as _time
 import math as _math
 
+# for cpyext, use these as base classes
+from __pypy__._pypydatetime import dateinterop, deltainterop, timeinterop
+
 def _cmp(x, y):
     return 0 if x == y else 1 if x > y else -1
 
@@ -316,7 +319,7 @@
 
     return q
 
-class timedelta:
+class timedelta(deltainterop):
     """Represent the difference between two datetime objects.
 
     Supported operators:
@@ -429,7 +432,7 @@
         if abs(d) > 999999999:
             raise OverflowError("timedelta # of days is too large: %d" % d)
 
-        self = object.__new__(cls)
+        self = deltainterop.__new__(cls)
         self._days = d
         self._seconds = s
         self._microseconds = us
@@ -638,7 +641,7 @@
                           microseconds=999999)
 timedelta.resolution = timedelta(microseconds=1)
 
-class date:
+class date(dateinterop):
     """Concrete date type.
 
     Constructors:
@@ -678,12 +681,12 @@
         if month is None and isinstance(year, bytes) and len(year) == 4 and \
                 1 <= year[2] <= 12:
             # Pickle support
-            self = object.__new__(cls)
+            self = dateinterop.__new__(cls)
             self.__setstate(year)
             self._hashcode = -1
             return self
         year, month, day = _check_date_fields(year, month, day)
-        self = object.__new__(cls)
+        self = dateinterop.__new__(cls)
         self._year = year
         self._month = month
         self._day = day
@@ -1008,7 +1011,7 @@
 
 _tzinfo_class = tzinfo
 
-class time:
+class time(timeinterop):
     """Time with time zone.
 
     Constructors:
@@ -1044,14 +1047,14 @@
         """
         if isinstance(hour, bytes) and len(hour) == 6 and hour[0] < 24:
             # Pickle support
-            self = object.__new__(cls)
+            self = timeinterop.__new__(cls)
             self.__setstate(hour, minute or None)
             self._hashcode = -1
             return self
         hour, minute, second, microsecond = _check_time_fields(
             hour, minute, second, microsecond)
         _check_tzinfo_arg(tzinfo)
-        self = object.__new__(cls)
+        self = timeinterop.__new__(cls)
         self._hour = hour
         self._minute = minute
         self._second = second
@@ -1329,7 +1332,7 @@
                 microsecond=0, tzinfo=None):
         if isinstance(year, bytes) and len(year) == 10 and 1 <= year[2] <= 12:
             # Pickle support
-            self = object.__new__(cls)
+            self = dateinterop.__new__(cls)
             self.__setstate(year, month)
             self._hashcode = -1
             return self
@@ -1337,7 +1340,7 @@
         hour, minute, second, microsecond = _check_time_fields(
             hour, minute, second, microsecond)
         _check_tzinfo_arg(tzinfo)
-        self = object.__new__(cls)
+        self = dateinterop.__new__(cls)
         self._year = year
         self._month = month
         self._day = day
diff --git a/lib-python/3/distutils/sysconfig_pypy.py b/lib-python/3/distutils/sysconfig_pypy.py
--- a/lib-python/3/distutils/sysconfig_pypy.py
+++ b/lib-python/3/distutils/sysconfig_pypy.py
@@ -63,39 +63,9 @@
 
 def _init_posix():
     """Initialize the module as appropriate for POSIX systems."""
-    so_ext = [s[0] for s in imp.get_suffixes() if s[2] == imp.C_EXTENSION][0]
-
-    g = {}
-    g['CC'] = "cc -pthread"
-    g['CXX'] = "c++ -pthread"
-    g['OPT'] = "-DNDEBUG -O2"
-    g['CFLAGS'] = "-DNDEBUG -O2"
-    g['CCSHARED'] = "-fPIC"
-    g['LDSHARED'] = "cc -pthread -shared"
-    g['EXT_SUFFIX'] = so_ext
-    g['SHLIB_SUFFIX'] = ".so"
-    g['SO'] = so_ext  # deprecated in Python 3, for backward compatibility
-    g['AR'] = "ar"
-    g['ARFLAGS'] = "rc"
-    g['EXE'] = ""
-    g['LIBDIR'] = os.path.join(sys.prefix, 'lib')
-    g['VERSION'] = get_python_version()
-
-    if sys.platform[:6] == "darwin":
-        import platform
-        if platform.machine() == 'i386':
-            if platform.architecture()[0] == '32bit':
-                arch = 'i386'
-            else:
-                arch = 'x86_64'
-        else:
-            # just a guess
-            arch = platform.machine()
-        g['LDSHARED'] += ' -undefined dynamic_lookup'
-        g['CC'] += ' -arch %s' % (arch,)
-
+    from _sysconfigdata import build_time_vars
     global _config_vars
-    _config_vars = g
+    _config_vars = build_time_vars
 
 
 def _init_nt():
@@ -221,4 +191,3 @@
 
 from .sysconfig_cpython import (
     parse_makefile, _variable_rx, expand_makefile_vars)
-
diff --git a/lib-python/3/fractions.py b/lib-python/3/fractions.py
--- a/lib-python/3/fractions.py
+++ b/lib-python/3/fractions.py
@@ -566,7 +566,7 @@
         else:
             return Fraction(round(self / shift) * shift)
 
-    def __hash__(self):
+    def __hash__(self, CACHE=[-1] * 1001):
         """hash(self)"""
 
         # XXX since this method is expensive, consider caching the result
@@ -580,12 +580,23 @@
         # dinv is the inverse of self._denominator modulo the prime
         # _PyHASH_MODULUS, or 0 if self._denominator is divisible by
         # _PyHASH_MODULUS.
-        dinv = pow(self._denominator, _PyHASH_MODULUS - 2, _PyHASH_MODULUS)
+
+        # PyPy3: added caching of pow() results for denominators up to 1000
+        denom = self._denominator
+        assert denom >= 0
+        if denom < len(CACHE):
+            dinv = CACHE[denom]
+            if dinv == -1:
+                dinv = pow(denom, _PyHASH_MODULUS - 2, _PyHASH_MODULUS)
+                CACHE[denom] = dinv
+        else:
+            dinv = pow(denom, _PyHASH_MODULUS - 2, _PyHASH_MODULUS)
+
         if not dinv:
             hash_ = _PyHASH_INF
         else:
             hash_ = abs(self._numerator) * dinv % _PyHASH_MODULUS
-        result = hash_ if self >= 0 else -hash_
+        result = hash_ if self._numerator >= 0 else -hash_
         return -2 if result == -1 else result
 
     def __eq__(a, b):
diff --git a/lib-python/3/hashlib.py b/lib-python/3/hashlib.py
--- a/lib-python/3/hashlib.py
+++ b/lib-python/3/hashlib.py
@@ -134,9 +134,14 @@
     __get_hash = __get_openssl_constructor
     algorithms_available = algorithms_available.union(
             _hashlib.openssl_md_meth_names)
-except ImportError:
+except ImportError as e:
     new = __py_new
     __get_hash = __get_builtin_constructor
+    # added by PyPy
+    import warnings
+    warnings.warn("The _hashlib module is not available, falling back "
+                  "to a much slower implementation (%s)" % str(e),
+                  RuntimeWarning)
 
 try:
     # OpenSSL's PKCS5_PBKDF2_HMAC requires OpenSSL 1.0+ with HMAC and SHA
diff --git a/lib-python/3/opcode.py b/lib-python/3/opcode.py
--- a/lib-python/3/opcode.py
+++ b/lib-python/3/opcode.py
@@ -223,5 +223,6 @@
 def_op('CALL_METHOD', 202)            # #args not including 'self'
 def_op('BUILD_LIST_FROM_ARG', 203)
 jrel_op('JUMP_IF_NOT_DEBUG', 204)     # jump over assert statements
+def_op('LOAD_REVDB_VAR', 205)         # reverse debugger (syntax example: $5)
 
 del def_op, name_op, jrel_op, jabs_op
diff --git a/lib-python/3/sysconfig.py b/lib-python/3/sysconfig.py
--- a/lib-python/3/sysconfig.py
+++ b/lib-python/3/sysconfig.py
@@ -20,30 +20,30 @@
 
 _INSTALL_SCHEMES = {
     'posix_prefix': {
-        'stdlib': '{installed_base}/lib/python{py_version_short}',
-        'platstdlib': '{platbase}/lib/python{py_version_short}',
-        'purelib': '{base}/lib/python{py_version_short}/site-packages',
-        'platlib': '{platbase}/lib/python{py_version_short}/site-packages',
+        'stdlib': '{installed_base}/lib/{implementation_lower}{py_version_short}',
+        'platstdlib': '{platbase}/lib/{implementation_lower}{py_version_short}',
+        'purelib': '{base}/lib/{implementation_lower}{py_version_short}/site-packages',
+        'platlib': '{platbase}/lib/{implementation_lower}{py_version_short}/site-packages',
         'include':
-            '{installed_base}/include/python{py_version_short}{abiflags}',
+            '{installed_base}/include/{implementation_lower}{py_version_short}{abiflags}',
         'platinclude':
-            '{installed_platbase}/include/python{py_version_short}{abiflags}',
+            '{installed_platbase}/include/{implementation_lower}{py_version_short}{abiflags}',
         'scripts': '{base}/bin',
         'data': '{base}',
         },
     'posix_home': {
-        'stdlib': '{installed_base}/lib/python',
-        'platstdlib': '{base}/lib/python',
-        'purelib': '{base}/lib/python',
-        'platlib': '{base}/lib/python',
-        'include': '{installed_base}/include/python',
-        'platinclude': '{installed_base}/include/python',
+        'stdlib': '{installed_base}/lib/{implementation_lower}',
+        'platstdlib': '{base}/lib/{implementation_lower}',
+        'purelib': '{base}/lib/{implementation_lower}',
+        'platlib': '{base}/lib/{implementation_lower}',
+        'include': '{installed_base}/include/{implementation_lower}',
+        'platinclude': '{installed_base}/include/{implementation_lower}',
         'scripts': '{base}/bin',
         'data': '{base}',
         },
     'pypy': {
-        'stdlib': '{installed_base}/lib-python',
-        'platstdlib': '{base}/lib-python',
+        'stdlib': '{installed_base}/lib-{implementation_lower}',
+        'platstdlib': '{base}/lib-{implementation_lower}',
         'purelib': '{base}/site-packages',
         'platlib': '{base}/site-packages',
         'include': '{installed_base}/include',
@@ -62,28 +62,28 @@
         'data': '{base}',
         },
     'nt_user': {
-        'stdlib': '{userbase}/Python{py_version_nodot}',
-        'platstdlib': '{userbase}/Python{py_version_nodot}',
-        'purelib': '{userbase}/Python{py_version_nodot}/site-packages',
-        'platlib': '{userbase}/Python{py_version_nodot}/site-packages',
-        'include': '{userbase}/Python{py_version_nodot}/Include',
-        'scripts': '{userbase}/Python{py_version_nodot}/Scripts',
+        'stdlib': '{userbase}/{implementation}{py_version_nodot}',
+        'platstdlib': '{userbase}/{implementation}{py_version_nodot}',
+        'purelib': '{userbase}/{implementation}{py_version_nodot}/site-packages',
+        'platlib': '{userbase}/{implementation}{py_version_nodot}/site-packages',
+        'include': '{userbase}/{implementation}{py_version_nodot}/Include',
+        'scripts': '{userbase}/{implementation}{py_version_nodot}/Scripts',
         'data': '{userbase}',
         },
     'posix_user': {
-        'stdlib': '{userbase}/lib/python{py_version_short}',
-        'platstdlib': '{userbase}/lib/python{py_version_short}',
-        'purelib': '{userbase}/lib/python{py_version_short}/site-packages',
-        'platlib': '{userbase}/lib/python{py_version_short}/site-packages',
-        'include': '{userbase}/include/python{py_version_short}',
+        'stdlib': '{userbase}/lib/{implementation_lower}{py_version_short}',
+        'platstdlib': '{userbase}/lib/{implementation_lower}{py_version_short}',
+        'purelib': '{userbase}/lib/{implementation_lower}{py_version_short}/site-packages',
+        'platlib': '{userbase}/lib/{implementation_lower}{py_version_short}/site-packages',
+        'include': '{userbase}/include/{implementation_lower}{py_version_short}',
         'scripts': '{userbase}/bin',
         'data': '{userbase}',
         },
     'osx_framework_user': {
-        'stdlib': '{userbase}/lib/python',
-        'platstdlib': '{userbase}/lib/python',
-        'purelib': '{userbase}/lib/python/site-packages',
-        'platlib': '{userbase}/lib/python/site-packages',
+        'stdlib': '{userbase}/lib/{implementation_lower}',
+        'platstdlib': '{userbase}/lib/{implementation_lower}',
+        'purelib': '{userbase}/lib/{implementation_lower}/site-packages',
+        'platlib': '{userbase}/lib/{implementation_lower}/site-packages',
         'include': '{userbase}/include',
         'scripts': '{userbase}/bin',
         'data': '{userbase}',
@@ -106,6 +106,11 @@
 _USER_BASE = None
 
 
+def _get_implementation():
+    if '__pypy__' in sys.builtin_module_names:
+        return 'PyPy'
+    return 'Python'
+
 def _safe_realpath(path):
     try:
         return realpath(path)
@@ -540,6 +545,8 @@
         except AttributeError:
             # sys.abiflags may not be defined on all platforms.
             _CONFIG_VARS['abiflags'] = ''
+        _CONFIG_VARS['implementation'] = _get_implementation()
+        _CONFIG_VARS['implementation_lower'] = _get_implementation().lower()
 
         if os.name == 'nt':
             _init_non_posix(_CONFIG_VARS)
@@ -715,6 +722,8 @@
     _print_dict('Paths', get_paths())
     print()
     _print_dict('Variables', get_config_vars())
+    print
+    _print_dict('User', get_paths('%s_user' % os.name))
 
 
 if __name__ == '__main__':
diff --git a/lib-python/3/test/test_exceptions.py b/lib-python/3/test/test_exceptions.py
--- a/lib-python/3/test/test_exceptions.py
+++ b/lib-python/3/test/test_exceptions.py
@@ -164,10 +164,10 @@
 
         is_pypy = check_impl_detail(pypy=True)
         check('def fact(x):\n\treturn x!\n', 2, 10)
-        check('1 +\n', 1, 4 - is_pypy)
-        check('def spam():\n  print(1)\n print(2)', 3, 0 if is_pypy else 10)
-        check('Python = "Python" +', 1, 20 - is_pypy)
-        check('Python = "\u1e54\xfd\u0163\u0125\xf2\xf1" +', 1, 20 - is_pypy)
+        check('1 +\n', 1, 4)
+        check('def spam():\n  print(1)\n print(2)', 3, 2 if is_pypy else 10)
+        check('Python = "Python" +', 1, 20)
+        check('Python = "\u1e54\xfd\u0163\u0125\xf2\xf1" +', 1, 20)
 
     @cpython_only
     def testSettingException(self):
diff --git a/lib-python/3/test/test_fstring.py b/lib-python/3/test/test_fstring.py
--- a/lib-python/3/test/test_fstring.py
+++ b/lib-python/3/test/test_fstring.py
@@ -319,7 +319,7 @@
                             ["f'{3)+(4}'",
                              ])
 
-        self.assertAllRaise(SyntaxError, 'EOL while scanning string literal',
+        self.assertAllRaise(SyntaxError, r'end of line \(EOL\) while scanning string literal',
                             ["f'{\n}'",
                              ])
 
@@ -741,7 +741,7 @@
         self.assertEqual('{d[0]}'.format(d=d), 'integer')
 
     def test_invalid_expressions(self):
-        self.assertAllRaise(SyntaxError, 'invalid syntax',
+        self.assertAllRaise(SyntaxError, "closing parenthesis '.' does not match opening parenthesis '.'",
                             [r"f'{a[4)}'",
                              r"f'{a(4]}'",
                             ])
diff --git a/lib-python/3/test/test_sysconfig.py b/lib-python/3/test/test_sysconfig.py
--- a/lib-python/3/test/test_sysconfig.py
+++ b/lib-python/3/test/test_sysconfig.py
@@ -4,6 +4,7 @@
 import subprocess
 import shutil
 from copy import copy
+from distutils.spawn import find_executable
 
 from test.support import (run_unittest, TESTFN, unlink, check_warnings,
                           captured_stdout, impl_detail, import_module,
@@ -297,6 +298,30 @@
 
         self.assertIn(ldflags, ldshared)
 
+    @unittest.skipIf(sys.platform == "win32", "Does not apply to Windows")
+    def test_cc_values(self):
+        """ CC and CXX should be set for pypy """
+        for var in ["CC", "CXX"]:
+            assert sysconfig.get_config_var(var) is not None
+
+    @unittest.skipIf(not find_executable("gcc"),
+        "Does not apply to machines without gcc installed"
+    )
+    def test_gcc_values(self):
+        """ if gcc is installed on the box, gcc values should be set. """
+        assert "gcc" in sysconfig.get_config_var("CC")
+        assert sysconfig.get_config_var("GNULD") == "yes"
+        assert "gcc" in sysconfig.get_config_var("LDSHARED")
+
+
+    @unittest.skipIf(not find_executable("g++"),
+        "Does not apply to machines without g++ installed"
+    )
+    def test_gplusplus_values(self):
+        """ if g++ is installed on the box, g++ values should be set. """
+        assert "g++" in sysconfig.get_config_var("CXX")
+
+
     @unittest.skipUnless(sys.platform == "darwin", "test only relevant on MacOSX")
     def test_platform_in_subprocess(self):
         my_platform = sysconfig.get_platform()
diff --git a/lib-python/3/test/test_sysconfig_pypy.py b/lib-python/3/test/test_sysconfig_pypy.py
new file mode 100644
--- /dev/null
+++ b/lib-python/3/test/test_sysconfig_pypy.py
@@ -0,0 +1,17 @@
+import os
+import sys
+import unittest
+import site
+
+
+class TestSysConfigPypy(unittest.TestCase):
+    def test_install_schemes(self):
+        # User-site etc. paths should have "pypy" and not "python"
+        # inside them.
+        if site.ENABLE_USER_SITE:
+            parts = site.USER_SITE.lower().split(os.path.sep)
+            assert any(x.startswith('pypy') for x in parts[-2:]), parts
+
+
+if __name__ == "__main__":
+    unittest.main()
diff --git a/lib-python/conftest.py b/lib-python/conftest.py
--- a/lib-python/conftest.py
+++ b/lib-python/conftest.py
@@ -422,6 +422,7 @@
     RegrTest('test_sys_setprofile.py', core=True),
     RegrTest('test_sys_settrace.py', core=True),
     RegrTest('test_sysconfig.py'),
+    RegrTest('test_sysconfig_pypy.py'),
     RegrTest('test_syslog.py'),
     RegrTest('test_tarfile.py'),
     RegrTest('test_tcl.py'),
diff --git a/lib_pypy/_cffi_ssl/README.md b/lib_pypy/_cffi_ssl/README.md
--- a/lib_pypy/_cffi_ssl/README.md
+++ b/lib_pypy/_cffi_ssl/README.md
@@ -14,6 +14,8 @@
 
 * ``_cffi_src/openssl/x509_vfy.py`` for issue #2605 (ca4d0c90f5a1)
 
+* ``_cffi_src/openssl/pypy_win32_extra.py`` for Win32-only functionality like ssl.enum_certificates()
+
 
 # Tests?
 
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/openssl/pypy_win32_extra.py b/lib_pypy/_cffi_ssl/_cffi_src/openssl/pypy_win32_extra.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/_cffi_src/openssl/pypy_win32_extra.py
@@ -0,0 +1,84 @@
+#
+# An extra bit of logic for the Win32-only functionality that is missing from the
+# version from cryptography.
+#
+
+import sys
+
+INCLUDES = """
+#include <Wincrypt.h>
+"""
+
+TYPES = """
+typedef ... *HCERTSTORE;
+typedef ... *HCRYPTPROV_LEGACY;
+
+typedef struct {
+    DWORD      dwCertEncodingType;
+    BYTE       *pbCertEncoded;
+    DWORD      cbCertEncoded;
+    ...;
+} CERT_CONTEXT, *PCCERT_CONTEXT;
+
+typedef struct {
+    DWORD      dwCertEncodingType;
+    BYTE       *pbCrlEncoded;
+    DWORD      cbCrlEncoded;
+    ...;
+} CRL_CONTEXT, *PCCRL_CONTEXT;
+
+typedef struct {
+    DWORD cUsageIdentifier;
+    LPSTR *rgpszUsageIdentifier;
+    ...;
+} CERT_ENHKEY_USAGE, *PCERT_ENHKEY_USAGE;
+"""
+
+FUNCTIONS = """
+HCERTSTORE WINAPI CertOpenStore(
+         LPCSTR            lpszStoreProvider,
+         DWORD             dwMsgAndCertEncodingType,
+         HCRYPTPROV_LEGACY hCryptProv,
+         DWORD             dwFlags,
+         const char        *pvPara
+);
+PCCERT_CONTEXT WINAPI CertEnumCertificatesInStore(
+         HCERTSTORE     hCertStore,
+         PCCERT_CONTEXT pPrevCertContext
+);
+BOOL WINAPI CertFreeCertificateContext(
+         PCCERT_CONTEXT pCertContext
+);
+BOOL WINAPI CertFreeCRLContext(
+         PCCRL_CONTEXT pCrlContext
+);
+BOOL WINAPI CertCloseStore(
+         HCERTSTORE hCertStore,
+         DWORD      dwFlags
+);
+BOOL WINAPI CertGetEnhancedKeyUsage(
+         PCCERT_CONTEXT     pCertContext,
+         DWORD              dwFlags,
+         PCERT_ENHKEY_USAGE pUsage,
+         DWORD              *pcbUsage
+);
+PCCRL_CONTEXT WINAPI CertEnumCRLsInStore(
+         HCERTSTORE    hCertStore,
+         PCCRL_CONTEXT pPrevCrlContext
+);
+"""
+
+MACROS = """
+#define CERT_STORE_READONLY_FLAG ...
+#define CERT_SYSTEM_STORE_LOCAL_MACHINE ...
+#define CRYPT_E_NOT_FOUND ...
+#define CERT_FIND_PROP_ONLY_ENHKEY_USAGE_FLAG ...
+#define CERT_FIND_EXT_ONLY_ENHKEY_USAGE_FLAG ...
+#define X509_ASN_ENCODING ...
+#define PKCS_7_ASN_ENCODING ...
+
+static const LPCSTR CERT_STORE_PROV_SYSTEM_A;
+"""
+
+CUSTOMIZATIONS = """
+"""
diff --git a/lib_pypy/_cffi_ssl/_stdssl/__init__.py b/lib_pypy/_cffi_ssl/_stdssl/__init__.py
--- a/lib_pypy/_cffi_ssl/_stdssl/__init__.py
+++ b/lib_pypy/_cffi_ssl/_stdssl/__init__.py
@@ -24,6 +24,7 @@
 from enum import IntEnum as _IntEnum
 
 if sys.platform == 'win32':
+    from _cffi_ssl._stdssl.win32_extra import enum_certificates, enum_crls
     HAVE_POLL = False
 else:
     from select import poll, POLLIN, POLLOUT
@@ -219,7 +220,6 @@
         if server_hostname:
             self.server_hostname = server_hostname.decode('idna', 'strict')
 
-        lib.ERR_get_state()
         lib.ERR_clear_error()
         self.ssl = ssl = ffi.gc(lib.SSL_new(ctx), lib.SSL_free)
 
diff --git a/lib_pypy/_cffi_ssl/_stdssl/win32_extra.py b/lib_pypy/_cffi_ssl/_stdssl/win32_extra.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/_stdssl/win32_extra.py
@@ -0,0 +1,101 @@
+from _pypy_openssl import lib, ffi
+
+
+def enum_certificates(store_name):
+    """Retrieve certificates from Windows' cert store.
+
+store_name may be one of 'CA', 'ROOT' or 'MY'.  The system may provide
+more cert storages, too.  The function returns a list of (bytes,
+encoding_type, trust) tuples.  The encoding_type flag can be interpreted
+with X509_ASN_ENCODING or PKCS_7_ASN_ENCODING. The trust setting is either
+a set of OIDs or the boolean True.
+    """
+    hStore = lib.CertOpenStore(lib.CERT_STORE_PROV_SYSTEM_A, 0, ffi.NULL,
+                               lib.CERT_STORE_READONLY_FLAG | lib.CERT_SYSTEM_STORE_LOCAL_MACHINE,
+                               bytes(store_name, "ascii"))
+    if hStore == ffi.NULL:
+        raise WindowsError(*ffi.getwinerror())
+    
+    result = []
+    pCertCtx = ffi.NULL
+    try:
+        while True:
+            pCertCtx = lib.CertEnumCertificatesInStore(hStore, pCertCtx)
+            if pCertCtx == ffi.NULL:
+                break
+            cert = ffi.buffer(pCertCtx.pbCertEncoded, pCertCtx.cbCertEncoded)[:]
+            enc = certEncodingType(pCertCtx.dwCertEncodingType)
+            keyusage = parseKeyUsage(pCertCtx, lib.CERT_FIND_PROP_ONLY_ENHKEY_USAGE_FLAG)
+            if keyusage is True:
+                keyusage = parseKeyUsage(pCertCtx, lib.CERT_FIND_EXT_ONLY_ENHKEY_USAGE_FLAG)
+            result.append((cert, enc, keyusage))
+    finally:
+        if pCertCtx != ffi.NULL:
+            lib.CertFreeCertificateContext(pCertCtx)
+        if not lib.CertCloseStore(hStore, 0):
+            # This error case might shadow another exception.
+            raise WindowsError(*ffi.getwinerror())
+    return result
+
+
+def enum_crls(store_name):
+    """Retrieve CRLs from Windows' cert store.
+
+store_name may be one of 'CA', 'ROOT' or 'MY'.  The system may provide
+more cert storages, too.  The function returns a list of (bytes,
+encoding_type) tuples.  The encoding_type flag can be interpreted with
+X509_ASN_ENCODING or PKCS_7_ASN_ENCODING."""
+    hStore = lib.CertOpenStore(lib.CERT_STORE_PROV_SYSTEM_A, 0, ffi.NULL,
+                               lib.CERT_STORE_READONLY_FLAG | lib.CERT_SYSTEM_STORE_LOCAL_MACHINE,
+                               bytes(store_name, "ascii"))
+    if hStore == ffi.NULL:
+        raise WindowsError(*ffi.getwinerror())
+
+    result = []
+    pCrlCtx = ffi.NULL
+    try:
+        while True:
+            pCrlCtx = lib.CertEnumCRLsInStore(hStore, pCrlCtx)
+            if pCrlCtx == ffi.NULL:
+                break
+            crl = ffi.buffer(pCrlCtx.pbCrlEncoded, pCrlCtx.cbCrlEncoded)[:]
+            enc = certEncodingType(pCrlCtx.dwCertEncodingType)
+            result.append((crl, enc))
+    finally:
+        if pCrlCtx != ffi.NULL:
+            lib.CertFreeCRLContext(pCrlCtx)
+        if not lib.CertCloseStore(hStore, 0):
+            # This error case might shadow another exception.
+            raise WindowsError(*ffi.getwinerror())
+    return result
+
+
+def certEncodingType(encodingType):
+    if encodingType == lib.X509_ASN_ENCODING:
+        return "x509_asn"
+    if encodingType == lib.PKCS_7_ASN_ENCODING:
+        return "pkcs_7_asn"
+    return encodingType
+
+def parseKeyUsage(pCertCtx, flags):
+    pSize = ffi.new("DWORD *")
+    if not lib.CertGetEnhancedKeyUsage(pCertCtx, flags, ffi.NULL, pSize):
+        error_with_message = ffi.getwinerror()
+        if error_with_message[0] == lib.CRYPT_E_NOT_FOUND:
+            return True
+        raise WindowsError(*error_with_message)
+
+    pUsageMem = ffi.new("char[]", pSize[0])
+    pUsage = ffi.cast("PCERT_ENHKEY_USAGE", pUsageMem)
+    if not lib.CertGetEnhancedKeyUsage(pCertCtx, flags, pUsage, pSize):
+        error_with_message = ffi.getwinerror()
+        if error_with_message[0] == lib.CRYPT_E_NOT_FOUND:
+            return True
+        raise WindowsError(*error_with_message)
+
+    retval = set()
+    for i in range(pUsage.cUsageIdentifier):
+        if pUsage.rgpszUsageIdentifier[i]:
+            oid = ffi.string(pUsage.rgpszUsageIdentifier[i]).decode('ascii')
+            retval.add(oid)
+    return retval
diff --git a/lib_pypy/_ctypes/array.py b/lib_pypy/_ctypes/array.py
--- a/lib_pypy/_ctypes/array.py
+++ b/lib_pypy/_ctypes/array.py
@@ -81,8 +81,11 @@
     def _CData_output(self, resarray, base=None, index=-1):
         from _rawffi.alt import types
         # If a char_p or unichar_p is received, skip the string interpretation
-        if base._ffiargtype != types.Pointer(types.char_p) and \
-           base._ffiargtype != types.Pointer(types.unichar_p):
+        try:
+            deref = type(base)._deref_ffiargtype()
+        except AttributeError:
+            deref = None
+        if deref != types.char_p and deref != types.unichar_p:
             # this seems to be a string if we're array of char, surprise!
             from ctypes import c_char, c_wchar
             if self._type_ is c_char:
@@ -127,6 +130,12 @@
                 value = self(*value)
         return _CDataMeta.from_param(self, value)
 
+    def _build_ffiargtype(self):
+        return _ffi.types.Pointer(self._type_.get_ffi_argtype())
+
+    def _deref_ffiargtype(self):
+        return self._type_.get_ffi_argtype()
+
 def array_get_slice_params(self, index):
     if hasattr(self, '_length_'):
         start, stop, step = index.indices(self._length_)
@@ -254,6 +263,5 @@
             _type_ = base
         )
         cls = ArrayMeta(name, (Array,), tpdict)
-        cls._ffiargtype = _ffi.types.Pointer(base.get_ffi_argtype())
         ARRAY_CACHE[key] = cls
         return cls
diff --git a/lib_pypy/_ctypes/basics.py b/lib_pypy/_ctypes/basics.py
--- a/lib_pypy/_ctypes/basics.py
+++ b/lib_pypy/_ctypes/basics.py
@@ -49,10 +49,13 @@
         else:
             return self.from_param(as_parameter)
 
+    def _build_ffiargtype(self):
+        return _shape_to_ffi_type(self._ffiargshape_)
+
     def get_ffi_argtype(self):
         if self._ffiargtype:
             return self._ffiargtype
-        self._ffiargtype = _shape_to_ffi_type(self._ffiargshape_)
+        self._ffiargtype = self._build_ffiargtype()
         return self._ffiargtype
 
     def _CData_output(self, resbuffer, base=None, index=-1):
diff --git a/lib_pypy/_ctypes/pointer.py b/lib_pypy/_ctypes/pointer.py
--- a/lib_pypy/_ctypes/pointer.py
+++ b/lib_pypy/_ctypes/pointer.py
@@ -70,7 +70,12 @@
         self._ffiarray = ffiarray
         self.__init__ = __init__
         self._type_ = TP
-        self._ffiargtype = _ffi.types.Pointer(TP.get_ffi_argtype())
+
+    def _build_ffiargtype(self):
+        return _ffi.types.Pointer(self._type_.get_ffi_argtype())
+
+    def _deref_ffiargtype(self):
+        return self._type_.get_ffi_argtype()
 
     from_address = cdata_from_address
 
diff --git a/lib_pypy/_ctypes/structure.py b/lib_pypy/_ctypes/structure.py
--- a/lib_pypy/_ctypes/structure.py
+++ b/lib_pypy/_ctypes/structure.py
@@ -160,6 +160,10 @@
             raise AttributeError("_fields_ is final")
         if self in [f[1] for f in value]:
             raise AttributeError("Structure or union cannot contain itself")
+        if self._ffiargtype is not None:
+            raise NotImplementedError("Too late to set _fields_: we already "
+                        "said to libffi that the structure type %s is opaque"
+                        % (self,))
         names_and_fields(
             self,
             value, self.__bases__[0],
diff --git a/lib_pypy/_ssl/__init__.py b/lib_pypy/_ssl/__init__.py
--- a/lib_pypy/_ssl/__init__.py
+++ b/lib_pypy/_ssl/__init__.py
@@ -16,12 +16,14 @@
     RAND_egd          = builtinify(RAND_egd)
 
 import sys
-if sys.platform == "win32" and 'enum_certificates' not in globals():
-    def enum_certificates(*args, **kwds):
-        import warnings
-        warnings.warn("ssl.enum_certificates() is not implemented")
-        return []
-    def enum_crls(*args, **kwds):
-        import warnings
-        warnings.warn("ssl.enum_crls() is not implemented")
-        return []
+if sys.platform == "win32":
+    if 'enum_certificates' not in globals():
+        def enum_certificates(*args, **kwds):
+            import warnings
+            warnings.warn("ssl.enum_certificates() is not implemented")
+            return []
+    if 'enum_crls' not in globals():
+        def enum_crls(*args, **kwds):
+            import warnings
+            warnings.warn("ssl.enum_crls() is not implemented")
+            return []
diff --git a/lib_pypy/_ssl_build.py b/lib_pypy/_ssl_build.py
--- a/lib_pypy/_ssl_build.py
+++ b/lib_pypy/_ssl_build.py
@@ -5,6 +5,11 @@
 from _cffi_ssl._cffi_src.build_openssl import (build_ffi_for_binding,
         _get_openssl_libraries, extra_link_args, compiler_type)
 
+if sys.platform == "win32":
+    pypy_win32_extra = ["pypy_win32_extra"]
+else:
+    pypy_win32_extra = []
+
 ffi = build_ffi_for_binding(
     module_name="_pypy_openssl",
     module_prefix="_cffi_src.openssl.",
@@ -44,10 +49,10 @@
         "x509_vfy",
         "pkcs7",
         "callbacks",
-    ],
+    ] + pypy_win32_extra,
     libraries=_get_openssl_libraries(sys.platform),
     extra_link_args=extra_link_args(compiler_type()),
 )
 
 if __name__ == '__main__':
-    ffi.compile()
+    ffi.compile(verbose=True)
diff --git a/lib_pypy/_sysconfigdata.py b/lib_pypy/_sysconfigdata.py
--- a/lib_pypy/_sysconfigdata.py
+++ b/lib_pypy/_sysconfigdata.py
@@ -1,10 +1,50 @@
 import _imp
+import os
+import sys
+from distutils.spawn import find_executable
 
 so_ext = _imp.extension_suffixes()[0]
 
+
 build_time_vars = {
-    "EXT_SUFFIX": so_ext,
-    "SHLIB_SUFFIX": so_ext,
     "SOABI": '-'.join(so_ext.split('.')[1].split('-')[:2]),
-    "SO": so_ext  # deprecated in Python 3, for backward compatibility
+    "SO": so_ext,  # deprecated in Python 3, for backward compatibility
+    'CC': "cc -pthread",
+    'CXX': "c++ -pthread",
+    'OPT': "-DNDEBUG -O2",
+    'CFLAGS': "-DNDEBUG -O2",
+    'CCSHARED': "-fPIC",
+    'LDSHARED': "cc -pthread -shared",
+    'EXT_SUFFIX': so_ext,
+    'SHLIB_SUFFIX': ".so",
+    'AR': "ar",
+    'ARFLAGS': "rc",
+    'EXE': "",
+    'LIBDIR': os.path.join(sys.prefix, 'lib'),
+    'VERSION': sys.version[:3]
 }
+
+if find_executable("gcc"):
+    build_time_vars.update({
+        "CC": "gcc -pthread",
+        "GNULD": "yes",
+        "LDSHARED": "gcc -pthread -shared",
+    })
+    if find_executable("g++"):
+        build_time_vars["CXX"] = "g++ -pthread"
+
+if sys.platform[:6] == "darwin":
+    import platform
+    if platform.machine() == 'i386':
+        if platform.architecture()[0] == '32bit':
+            arch = 'i386'
+        else:
+            arch = 'x86_64'
+    else:
+        # just a guess
+        arch = platform.machine()
+    build_time_vars['LDSHARED'] += ' -undefined dynamic_lookup'
+    build_time_vars['CC'] += ' -arch %s' % (arch,)
+    if "CXX" in build_time_vars:
+        build_time_vars['CXX'] += ' -arch %s' % (arch,)
+
diff --git a/lib_pypy/_winapi.py b/lib_pypy/_winapi.py
--- a/lib_pypy/_winapi.py
+++ b/lib_pypy/_winapi.py
@@ -98,9 +98,11 @@
     def GetOverlappedResult(self, wait):
         transferred = _ffi.new('DWORD[1]', [0])
         res = _kernel32.GetOverlappedResult(self.handle, self.overlapped, transferred, wait != 0)
-        if not res:
-            res = GetLastError()
-        if res in (ERROR_SUCCESS, ERROR_MORE_DATA, ERROR_OPERATION_ABORTED):
+        if res:
+            err = ERROR_SUCCESS
+        else:
+            err = GetLastError()
+        if err in (ERROR_SUCCESS, ERROR_MORE_DATA, ERROR_OPERATION_ABORTED):
             self.completed = 1
             self.pending = 0
         elif res == ERROR_IO_INCOMPLETE:
@@ -133,7 +135,7 @@
         assert success == 0
         err = _kernel32.GetLastError()
         if err == ERROR_IO_PENDING:
-            overlapped[0].pending = 1
+            ov.pending = 1
         elif err == ERROR_PIPE_CONNECTED:
             _kernel32.SetEvent(ov.overlapped[0].hEvent)
         else:
diff --git a/lib_pypy/cffi.egg-info/PKG-INFO b/lib_pypy/cffi.egg-info/PKG-INFO
--- a/lib_pypy/cffi.egg-info/PKG-INFO
+++ b/lib_pypy/cffi.egg-info/PKG-INFO
@@ -1,6 +1,6 @@
 Metadata-Version: 1.1
 Name: cffi
-Version: 1.11.5
+Version: 1.12.0
 Summary: Foreign Function Interface for Python calling C code.
 Home-page: http://cffi.readthedocs.org
 Author: Armin Rigo, Maciej Fijalkowski
diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py
--- a/lib_pypy/cffi/__init__.py
+++ b/lib_pypy/cffi/__init__.py
@@ -4,8 +4,8 @@
 from .api import FFI
 from .error import CDefError, FFIError, VerificationError, VerificationMissing
 
-__version__ = "1.11.5"
-__version_info__ = (1, 11, 5)
+__version__ = "1.12.0"
+__version_info__ = (1, 12, 0)
 
 # The verifier module file names are based on the CRC32 of a string that
 # contains the following version number.  It may be older than __version__
diff --git a/lib_pypy/cffi/_embedding.h b/lib_pypy/cffi/_embedding.h
--- a/lib_pypy/cffi/_embedding.h
+++ b/lib_pypy/cffi/_embedding.h
@@ -221,7 +221,7 @@
 
         if (f != NULL && f != Py_None) {
             PyFile_WriteString("\nFrom: " _CFFI_MODULE_NAME
-                               "\ncompiled with cffi version: 1.11.5"
+                               "\ncompiled with cffi version: 1.12.0"
                                "\n_cffi_backend module: ", f);
             modules = PyImport_GetModuleDict();
             mod = PyDict_GetItemString(modules, "_cffi_backend");
diff --git a/lib_pypy/cffi/api.py b/lib_pypy/cffi/api.py
--- a/lib_pypy/cffi/api.py
+++ b/lib_pypy/cffi/api.py
@@ -96,18 +96,21 @@
             self.CData, self.CType = backend._get_types()
         self.buffer = backend.buffer
 
-    def cdef(self, csource, override=False, packed=False):
+    def cdef(self, csource, override=False, packed=False, pack=None):
         """Parse the given C source.  This registers all declared functions,
         types, and global variables.  The functions and global variables can
         then be accessed via either 'ffi.dlopen()' or 'ffi.verify()'.
         The types can be used in 'ffi.new()' and other functions.
         If 'packed' is specified as True, all structs declared inside this
         cdef are packed, i.e. laid out without any field alignment at all.
+        Alternatively, 'pack' can be a small integer, and requests for
+        alignment greater than that are ignored (pack=1 is equivalent to
+        packed=True).
         """
-        self._cdef(csource, override=override, packed=packed)
+        self._cdef(csource, override=override, packed=packed, pack=pack)
 
-    def embedding_api(self, csource, packed=False):
-        self._cdef(csource, packed=packed, dllexport=True)
+    def embedding_api(self, csource, packed=False, pack=None):
+        self._cdef(csource, packed=packed, pack=pack, dllexport=True)
         if self._embedding is None:
             self._embedding = ''
 
diff --git a/lib_pypy/cffi/backend_ctypes.py b/lib_pypy/cffi/backend_ctypes.py
--- a/lib_pypy/cffi/backend_ctypes.py
+++ b/lib_pypy/cffi/backend_ctypes.py
@@ -730,7 +730,8 @@
         return self._new_struct_or_union('union', name, ctypes.Union)
 
     def complete_struct_or_union(self, CTypesStructOrUnion, fields, tp,
-                                 totalsize=-1, totalalignment=-1, sflags=0):
+                                 totalsize=-1, totalalignment=-1, sflags=0,
+                                 pack=0):
         if totalsize >= 0 or totalalignment >= 0:
             raise NotImplementedError("the ctypes backend of CFFI does not support "
                                       "structures completed by verify(); please "
@@ -751,6 +752,8 @@
                 bfield_types[fname] = Ellipsis
         if sflags & 8:
             struct_or_union._pack_ = 1
+        elif pack:
+            struct_or_union._pack_ = pack
         struct_or_union._fields_ = cfields
         CTypesStructOrUnion._bfield_types = bfield_types
         #
diff --git a/lib_pypy/cffi/cparser.py b/lib_pypy/cffi/cparser.py
--- a/lib_pypy/cffi/cparser.py
+++ b/lib_pypy/cffi/cparser.py
@@ -306,11 +306,25 @@
             msg = 'parse error\n%s' % (msg,)
         raise CDefError(msg)
 
-    def parse(self, csource, override=False, packed=False, dllexport=False):
+    def parse(self, csource, override=False, packed=False, pack=None,
+                    dllexport=False):
+        if packed:
+            if packed != True:
+                raise ValueError("'packed' should be False or True; use "
+                                 "'pack' to give another value")
+            if pack:
+                raise ValueError("cannot give both 'pack' and 'packed'")
+            pack = 1
+        elif pack:
+            if pack & (pack - 1):
+                raise ValueError("'pack' must be a power of two, not %r" %
+                    (pack,))
+        else:
+            pack = 0
         prev_options = self._options
         try:
             self._options = {'override': override,
-                             'packed': packed,
+                             'packed': pack,
                              'dllexport': dllexport}
             self._internal_parse(csource)
         finally:
diff --git a/lib_pypy/cffi/model.py b/lib_pypy/cffi/model.py
--- a/lib_pypy/cffi/model.py
+++ b/lib_pypy/cffi/model.py
@@ -342,7 +342,7 @@
     fixedlayout = None
     completed = 0
     partial = False
-    packed = False
+    packed = 0
 
     def __init__(self, name, fldnames, fldtypes, fldbitsize, fldquals=None):
         self.name = name
@@ -414,11 +414,14 @@
             fldtypes = [tp.get_cached_btype(ffi, finishlist)
                         for tp in self.fldtypes]
             lst = list(zip(self.fldnames, fldtypes, self.fldbitsize))
-            sflags = 0
+            extra_flags = ()
             if self.packed:
-                sflags = 8    # SF_PACKED
+                if self.packed == 1:
+                    extra_flags = (8,)    # SF_PACKED
+                else:
+                    extra_flags = (0, self.packed)
             ffi._backend.complete_struct_or_union(BType, lst, self,
-                                                  -1, -1, sflags)
+                                                  -1, -1, *extra_flags)
             #
         else:
             fldtypes = []
diff --git a/lib_pypy/cffi/recompiler.py b/lib_pypy/cffi/recompiler.py
--- a/lib_pypy/cffi/recompiler.py
+++ b/lib_pypy/cffi/recompiler.py
@@ -893,6 +893,12 @@
             else:
                 flags.append("_CFFI_F_CHECK_FIELDS")
             if tp.packed:
+                if tp.packed > 1:
+                    raise NotImplementedError(
+                        "%r is declared with 'pack=%r'; only 0 or 1 are "
+                        "supported in API mode (try to use \"...;\", which "
+                        "does not require a 'pack' declaration)" %
+                        (tp, tp.packed))
                 flags.append("_CFFI_F_PACKED")
         else:
             flags.append("_CFFI_F_EXTERNAL")
diff --git a/lib_pypy/grp.py b/lib_pypy/grp.py
--- a/lib_pypy/grp.py
+++ b/lib_pypy/grp.py
@@ -5,6 +5,8 @@
 import os
 from _pwdgrp_cffi import ffi, lib
 import _structseq
+import thread
+_lock = thread.allocate_lock()
 
 try: from __pypy__ import builtinify
 except ImportError: builtinify = lambda f: f
@@ -33,31 +35,34 @@
 
 @builtinify
 def getgrgid(gid):
-    res = lib.getgrgid(gid)
-    if not res:
-        # XXX maybe check error eventually
-        raise KeyError(gid)
-    return _group_from_gstruct(res)
+    with _lock:
+        res = lib.getgrgid(gid)
+        if not res:
+            # XXX maybe check error eventually
+            raise KeyError(gid)
+        return _group_from_gstruct(res)
 
 @builtinify
 def getgrnam(name):
     if not isinstance(name, str):
         raise TypeError("expected string")
-    res = lib.getgrnam(os.fsencode(name))
-    if not res:
-        raise KeyError("'getgrnam(): name not found: %s'" % name)
-    return _group_from_gstruct(res)
+    with _lock:
+        res = lib.getgrnam(os.fsencode(name))
+        if not res:
+            raise KeyError("'getgrnam(): name not found: %s'" % name)
+        return _group_from_gstruct(res)
 
 @builtinify
 def getgrall():
-    lib.setgrent()
     lst = []
-    while 1:
-        p = lib.getgrent()
-        if not p:
-            break
-        lst.append(_group_from_gstruct(p))
-    lib.endgrent()
+    with _lock:
+        lib.setgrent()
+        while 1:
+            p = lib.getgrent()
+            if not p:
+                break
+            lst.append(_group_from_gstruct(p))
+        lib.endgrent()
     return lst
 
 __all__ = ('struct_group', 'getgrgid', 'getgrnam', 'getgrall')
diff --git a/lib_pypy/pwd.py b/lib_pypy/pwd.py
--- a/lib_pypy/pwd.py
+++ b/lib_pypy/pwd.py
@@ -12,6 +12,8 @@
 
 from _pwdgrp_cffi import ffi, lib
 import _structseq
+import thread
+_lock = thread.allocate_lock()
 
 try: from __pypy__ import builtinify
 except ImportError: builtinify = lambda f: f
@@ -54,10 +56,11 @@
     Return the password database entry for the given numeric user ID.
     See pwd.__doc__ for more on password database entries.
     """
-    pw = lib.getpwuid(uid)
-    if not pw:
-        raise KeyError("getpwuid(): uid not found: %s" % uid)
-    return _mkpwent(pw)
+    with _lock:
+        pw = lib.getpwuid(uid)
+        if not pw:
+            raise KeyError("getpwuid(): uid not found: %s" % uid)
+        return _mkpwent(pw)
 
 @builtinify
 def getpwnam(name):
@@ -70,10 +73,11 @@
     if not isinstance(name, basestring):
         raise TypeError("expected string")
     name = str(name)
-    pw = lib.getpwnam(name)
-    if not pw:
-        raise KeyError("getpwname(): name not found: %s" % name)
-    return _mkpwent(pw)
+    with _lock:
+        pw = lib.getpwnam(name)
+        if not pw:
+            raise KeyError("getpwname(): name not found: %s" % name)
+        return _mkpwent(pw)
 
 @builtinify
 def getpwall():
@@ -83,13 +87,14 @@
     See pwd.__doc__ for more on password database entries.
     """
     users = []
-    lib.setpwent()
-    while True:
-        pw = lib.getpwent()
-        if not pw:
-            break
-        users.append(_mkpwent(pw))
-    lib.endpwent()
+    with _lock:
+        lib.setpwent()
+        while True:
+            pw = lib.getpwent()
+            if not pw:
+                break
+            users.append(_mkpwent(pw))
+        lib.endpwent()
     return users
 
 __all__ = ('struct_passwd', 'getpwuid', 'getpwnam', 'getpwall')
diff --git a/lib_pypy/pyrepl/simple_interact.py b/lib_pypy/pyrepl/simple_interact.py
--- a/lib_pypy/pyrepl/simple_interact.py
+++ b/lib_pypy/pyrepl/simple_interact.py
@@ -66,6 +66,10 @@
 
     while 1:
         try:
+            try:
+                sys.stdout.flush()
+            except:
+                pass
             ps1 = getattr(sys, 'ps1', '>>> ')
             ps2 = getattr(sys, 'ps2', '... ')
             try:
diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py
--- a/pypy/config/pypyoption.py
+++ b/pypy/config/pypyoption.py
@@ -39,18 +39,14 @@
     "thread", "itertools", "pyexpat", "cpyext", "array",
     "binascii", "_multiprocessing", '_warnings', "_collections",
     "_multibytecodec", "_continuation", "_cffi_backend",
-    "_csv", "_pypyjson", "_posixsubprocess", # "_cppyy", "micronumpy"
+    "_csv", "_pypyjson", "_posixsubprocess", "_cppyy", # "micronumpy",
     "_jitlog",
 ])
 
-from rpython.jit.backend import detect_cpu
-try:
-    if detect_cpu.autodetect().startswith('x86'):
-        if not sys.platform.startswith('openbsd'):
-            working_modules.add('_vmprof')
-            working_modules.add('faulthandler')
-except detect_cpu.ProcessorAutodetectError:
-    pass
+import rpython.rlib.rvmprof.cintf
+if rpython.rlib.rvmprof.cintf.IS_SUPPORTED:
+    working_modules.add('_vmprof')
+    working_modules.add('faulthandler')
 
 translation_modules = default_modules.copy()
 translation_modules.update([
@@ -61,6 +57,11 @@
     "termios", "_minimal_curses",
 ])
 
+reverse_debugger_disable_modules = set([
+    "_continuation", "_vmprof", "_multiprocessing",
+    "micronumpy",
+    ])
+
 # XXX this should move somewhere else, maybe to platform ("is this posixish"
 #     check or something)
 if sys.platform == "win32":
@@ -301,6 +302,9 @@
     modules = working_modules.copy()
     if config.translation.sandbox:
         modules = default_modules
+    if config.translation.reverse_debugger:
+        for mod in reverse_debugger_disable_modules:
+            setattr(config.objspace.usemodules, mod, False)
     # ignore names from 'essential_modules', notably 'exceptions', which
     # may not be present in config.objspace.usemodules at all
     modules = [name for name in modules if name not in essential_modules]
@@ -319,3 +323,4 @@
     parser = to_optparse(config) #, useoptions=["translation.*"])
     option, args = parser.parse_args()
     print config
+    print working_modules
diff --git a/pypy/doc/architecture.rst b/pypy/doc/architecture.rst
--- a/pypy/doc/architecture.rst
+++ b/pypy/doc/architecture.rst
@@ -73,3 +73,63 @@
 This division between bytecode evaluator and object space gives a lot of
 flexibility.  One can plug in different :doc:`object spaces <objspace>` to get
 different or enriched behaviours of the Python objects.
+
+Layers
+------
+
+RPython
+~~~~~~~
+:ref:`RPython <rpython:language>` is the language in which we write interpreters.
+Not the entire PyPy project is written in RPython, only the parts that are
+compiled in the translation process. The interesting point is that RPython
+has no parser, it's compiled from the live python objects, which makes it
+possible to do all kinds of metaprogramming during import time. In short,
+Python is a meta programming language for RPython.
+
+The RPython standard library is to be found in the ``rlib`` subdirectory.
+
+Consult `Getting Started with RPython`_ for further reading
+
+Translation
+~~~~~~~~~~~
+The translation toolchain - this is the part that takes care of translating
+RPython to flow graphs and then to C. There is more in the
+:doc:`architecture <architecture>` document written about it.
+
+It lives in the ``rpython`` directory: ``flowspace``, ``annotator``
+and ``rtyper``.
+
+PyPy Interpreter
+~~~~~~~~~~~~~~~~
+This is in the ``pypy`` directory.  ``pypy/interpreter`` is a standard
+interpreter for Python written in RPython.  The fact that it is
+RPython is not apparent at first.  Built-in modules are written in
+``pypy/module/*``.  Some modules that CPython implements in C are
+simply written in pure Python; they are in the top-level ``lib_pypy``
+directory.  The standard library of Python (with a few changes to
+accomodate PyPy) is in ``lib-python``.
+
+JIT Compiler
+~~~~~~~~~~~~
+:ref:`Just-in-Time Compiler (JIT) <rpython:jit>`: we have a tracing JIT that traces the
+interpreter written in RPython, rather than the user program that it
+interprets.  As a result it applies to any interpreter, i.e. any
+language.  But getting it to work correctly is not trivial: it
+requires a small number of precise "hints" and possibly some small
+refactorings of the interpreter.  The JIT itself also has several
+almost-independent parts: the tracer itself in ``rpython/jit/metainterp``, the
+optimizer in ``rpython/jit/metainterp/optimizer`` that optimizes a list of
+residual operations, and the backend in ``rpython/jit/backend/<machine-name>``
+that turns it into machine code.  Writing a new backend is a
+traditional way to get into the project.
+
+Garbage Collectors
+~~~~~~~~~~~~~~~~~~
+Garbage Collectors (GC): as you may notice if you are used to CPython's
+C code, there are no ``Py_INCREF/Py_DECREF`` equivalents in RPython code.
+:ref:`rpython:garbage-collection` is inserted
+during translation.  Moreover, this is not reference counting; it is a real
+GC written as more RPython code.  The best one we have so far is in
+``rpython/memory/gc/incminimark.py``.
+
+.. _`Getting started with RPython`: http://rpython.readthedocs.org/en/latest/getting-started.html
diff --git a/pypy/doc/build.rst b/pypy/doc/build.rst
--- a/pypy/doc/build.rst
+++ b/pypy/doc/build.rst
@@ -267,14 +267,14 @@
 * PyPy 2.5.1 or earlier: normal users would see permission errors.
   Installers need to run ``pypy -c "import gdbm"`` and other similar
   commands at install time; the exact list is in
-  :source:`pypy/tool/release/package.py <package.py>`.  Users
+  :source:`pypy/tool/release/package.py`.  Users
   seeing a broken installation of PyPy can fix it after-the-fact if they
   have sudo rights, by running once e.g. ``sudo pypy -c "import gdbm``.
 
 * PyPy 2.6 and later: anyone would get ``ImportError: no module named
   _gdbm_cffi``.  Installers need to run ``pypy _gdbm_build.py`` in the
   ``lib_pypy`` directory during the installation process (plus others;
-  see the exact list in :source:`pypy/tool/release/package.py <package.py>`).
+  see the exact list in :source:`pypy/tool/release/package.py`).
   Users seeing a broken
   installation of PyPy can fix it after-the-fact, by running ``pypy
   /path/to/lib_pypy/_gdbm_build.py``.  This command produces a file
diff --git a/pypy/doc/coding-guide.rst b/pypy/doc/coding-guide.rst
--- a/pypy/doc/coding-guide.rst
+++ b/pypy/doc/coding-guide.rst
@@ -539,7 +539,7 @@
 
     hg help branch
 
-.. _official wiki: http://mercurial.selenic.com/wiki/Branch
+.. _official wiki: https://www.mercurial-scm.org/wiki/
 
 
 .. _using-development-tracker:
@@ -547,15 +547,7 @@
 Using the development bug/feature tracker
 -----------------------------------------
 
-We have a `development tracker`_, based on Richard Jones'
-`roundup`_ application.  You can file bugs,
-feature requests or see what's going on
-for the next milestone, both from an E-Mail and from a
-web interface.
-
-.. _development tracker: https://bugs.pypy.org/
-.. _roundup: http://roundup.sourceforge.net/
-
+We use bitbucket for :source:`issues` tracking and :source:`pull-requests`.
 
 .. _testing:
 
diff --git a/pypy/doc/commandline_ref.rst b/pypy/doc/commandline_ref.rst
--- a/pypy/doc/commandline_ref.rst
+++ b/pypy/doc/commandline_ref.rst
@@ -8,3 +8,4 @@
    :maxdepth: 1
 
    man/pypy.1.rst
+   man/pypy3.1.rst
diff --git a/pypy/doc/conf.py b/pypy/doc/conf.py
--- a/pypy/doc/conf.py
+++ b/pypy/doc/conf.py
@@ -66,9 +66,9 @@
 # built documents.
 #
 # The short X.Y version.
-version = '5.8'
+version = '6.0'
 # The full version, including alpha/beta/rc tags.
-release = '5.8.0'
+release = '6.0.0'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.
diff --git a/pypy/doc/config/objspace.disable_entrypoints.txt b/pypy/doc/config/objspace.disable_entrypoints.txt
new file mode 100644
diff --git a/pypy/doc/config/objspace.fstrings.txt b/pypy/doc/config/objspace.fstrings.txt
new file mode 100644
diff --git a/pypy/doc/config/objspace.hash.txt b/pypy/doc/config/objspace.hash.txt
new file mode 100644
diff --git a/pypy/doc/config/objspace.usemodules._cppyy.txt b/pypy/doc/config/objspace.usemodules._cppyy.txt
new file mode 100644
--- /dev/null
+++ b/pypy/doc/config/objspace.usemodules._cppyy.txt
@@ -0,0 +1,1 @@
+The internal backend for cppyy
diff --git a/pypy/doc/config/objspace.usemodules._frozen_importlib.txt b/pypy/doc/config/objspace.usemodules._frozen_importlib.txt
new file mode 100644
diff --git a/pypy/doc/config/objspace.usemodules._jitlog.txt b/pypy/doc/config/objspace.usemodules._jitlog.txt
new file mode 100644
diff --git a/pypy/doc/config/objspace.usemodules._rawffi.txt b/pypy/doc/config/objspace.usemodules._rawffi.txt
--- a/pypy/doc/config/objspace.usemodules._rawffi.txt
+++ b/pypy/doc/config/objspace.usemodules._rawffi.txt
@@ -1,3 +1,3 @@
-An experimental module providing very low-level interface to
+A module providing very low-level interface to
 C-level libraries, for use when implementing ctypes, not
-intended for a direct use at all.
\ No newline at end of file
+intended for a direct use at all.
diff --git a/pypy/doc/config/objspace.usemodules.cpyext.txt b/pypy/doc/config/objspace.usemodules.cpyext.txt
--- a/pypy/doc/config/objspace.usemodules.cpyext.txt
+++ b/pypy/doc/config/objspace.usemodules.cpyext.txt
@@ -1,1 +1,1 @@
-Use (experimental) cpyext module, that tries to load and run CPython extension modules
+Use cpyext module to load and run CPython extension modules
diff --git a/pypy/doc/config/objspace.usemodules.faulthandler.txt b/pypy/doc/config/objspace.usemodules.faulthandler.txt
new file mode 100644
diff --git a/pypy/doc/config/translation.backendopt.replace_we_are_jitted.txt b/pypy/doc/config/translation.backendopt.replace_we_are_jitted.txt
new file mode 100644
diff --git a/pypy/doc/config/translation.jit_opencoder_model.txt b/pypy/doc/config/translation.jit_opencoder_model.txt
new file mode 100644
diff --git a/pypy/doc/config/translation.keepgoing.txt b/pypy/doc/config/translation.keepgoing.txt
new file mode 100644
diff --git a/pypy/doc/config/translation.libname.txt b/pypy/doc/config/translation.libname.txt
new file mode 100644
diff --git a/pypy/doc/config/translation.lto.txt b/pypy/doc/config/translation.lto.txt
new file mode 100644
diff --git a/pypy/doc/config/translation.profoptargs.txt b/pypy/doc/config/translation.profoptargs.txt
new file mode 100644
diff --git a/pypy/doc/config/translation.reverse_debugger.txt b/pypy/doc/config/translation.reverse_debugger.txt
new file mode 100644
diff --git a/pypy/doc/config/translation.split_gc_address_space.txt b/pypy/doc/config/translation.split_gc_address_space.txt
new file mode 100644
diff --git a/pypy/doc/contributing.rst b/pypy/doc/contributing.rst
new file mode 100644
--- /dev/null
+++ b/pypy/doc/contributing.rst
@@ -0,0 +1,472 @@
+Contributing Guidelines
+===========================
+
+.. contents::
+
+PyPy is a very large project that has a reputation of being hard to dive into.
+Some of this fame is warranted, some of it is purely accidental. There are three
+important lessons that everyone willing to contribute should learn:
+
+* PyPy has layers. There are many pieces of architecture that are very well
+  separated from each other. More about this below, but often the manifestation
+  of this is that things are at a different layer than you would expect them
+  to be. For example if you are looking for the JIT implementation, you will
+  not find it in the implementation of the Python programming language.
+
+* Because of the above, we are very serious about Test Driven Development.
+  It's not only what we believe in, but also that PyPy's architecture is
+  working very well with TDD in mind and not so well without it. Often
+  development means progressing in an unrelated corner, one unittest
+  at a time; and then flipping a giant switch, bringing it all together.
+  (It generally works out of the box.  If it doesn't, then we didn't
+  write enough unit tests.)  It's worth repeating - PyPy's
+  approach is great if you do TDD, and not so great otherwise.
+
+* PyPy uses an entirely different set of tools - most of them included
+  in the PyPy repository. There is no Makefile, nor autoconf. More below.
+
+The first thing to remember is that PyPy project is very different than most
+projects out there. It's also different from a classic compiler project,
+so academic courses about compilers often don't apply or lead in the wrong
+direction. However, if you want to understand how designing & building a runtime
+works in the real world then this is a great project!
+
+Getting involved
+^^^^^^^^^^^^^^^^
+
+PyPy employs a relatively standard open-source development process. You are
+encouraged as a first step to join our `pypy-dev mailing list`_ and IRC channel,
+details of which can be found in our :ref:`contact <contact>` section. The folks
+there are very friendly, and can point you in the right direction.
+
+We give out commit rights usually fairly liberally, so if you want to do something
+with PyPy, you can become a committer. We also run frequent coding sprints which
+are separately announced and often happen around Python conferences such as
+EuroPython or PyCon. Upcoming events are usually announced on `the blog`_.
+
+Further Reading: :ref:`Contact <contact>`
+
+.. _the blog: http://morepypy.blogspot.com
+.. _pypy-dev mailing list: http://mail.python.org/mailman/listinfo/pypy-dev
+
+
+Your first contribution
+^^^^^^^^^^^^^^^^^^^^^^^
+
+The first and most important rule how **not** to contribute to PyPy is
+"just hacking a feature". This won't work, and you'll find your PR will typically
+require a lot of re-work. There are a few reasons why not:
+
+* build times are large
+* PyPy has very thick layer separation
+* context of the cPython runtime is often required
+
+Instead, reach out on the dev mailing list or the IRC channel, and we're more
+than happy to help! :)
+
+Some ideas for first contributions are:
+
+* Documentation - this will give you an understanding of the pypy architecture
+* Test failures - find a failing test in the `nightly builds`_, and fix it
+* Missing language features - these are listed in our `issue tracker`_
+
+.. _nightly builds: http://buildbot.pypy.org/nightly/
+.. _issue tracker: https://bitbucket.org/pypy/pypy/issues
+
+Source Control
+--------------
+
+PyPy development is based a typical fork/pull request based workflow, centered
+around Mercurial (hg), hosted on Bitbucket. If you have not used this workflow
+before, a good introduction can be found here:
+
+    https://www.atlassian.com/git/tutorials/comparing-workflows/forking-workflow
+
+The cycle for a new PyPy contributor goes typically like this:
+
+Fork & Clone
+------------
+
+* Make an account on bitbucket_.
+
+* Go to https://bitbucket.org/pypy/pypy/ and click "fork" (left
+  icons).  You get a fork of the repository, e.g. in
+  `https://bitbucket.org/yourname/pypy/`.
+
+* Clone your new repo (i.e. the fork) to your local machine with the command
+  ``hg clone ssh://hg at bitbucket.org/yourname/pypy``.  It is a very slow
+  operation but only ever needs to be done once.  See also
+  http://pypy.org/download.html#building-from-source .
+  If you already cloned
+  ``https://bitbucket.org/pypy/pypy`` before, even if some time ago,
+  then you can reuse the same clone by editing the file ``.hg/hgrc`` in
+  your clone to contain the line ``default =
+  ssh://hg at bitbucket.org/yourname/pypy``, and then do ``hg pull && hg
+  up``.  If you already have such a clone but don't want to change it,
+  you can clone that copy with ``hg clone /path/to/other/copy``, and
+  then edit ``.hg/hgrc`` as above and do ``hg pull && hg up``.
+
+* Now you have a complete copy of the PyPy repo.  Make a branch
+  with a command like ``hg branch name_of_your_branch``.
+
+Edit
+----
+
+* Edit things.  Use ``hg diff`` to see what you changed.  Use ``hg add``
+  to make Mercurial aware of new files you added, e.g. new test files.
+  Use ``hg status`` to see if there are such files.  Write and run tests!
+  (See the rest of this page.)
+
+* Commit regularly with ``hg commit``.  A one-line commit message is
+  fine.  We love to have tons of commits; make one as soon as you have
+  some progress, even if it is only some new test that doesn't pass yet,
+  or fixing things even if not all tests pass.  Step by step, you are
+  building the history of your changes, which is the point of a version
+  control system.  (There are commands like ``hg log`` and ``hg up``
+  that you should read about later, to learn how to navigate this
+  history.)
+
+* The commits stay on your machine until you do ``hg push`` to "push"
+  them back to the repo named in the file ``.hg/hgrc``.  Repos are
+  basically just collections of commits (a commit is also called a
+  changeset): there is one repo per url, plus one for each local copy on
+  each local machine.  The commands ``hg push`` and ``hg pull`` copy
+  commits around, with the goal that all repos in question end up with
+  the exact same set of commits.  By opposition, ``hg up`` only updates
+  the "working copy" by reading the local repository, i.e. it makes the
+  files that you see correspond to the latest (or any other) commit
+  locally present.
+
+* You should push often; there is no real reason not to.  Remember that
+  even if they are pushed, with the setup above, the commits are (1)
+  only in ``bitbucket.org/yourname/pypy``, and (2) in the branch you
+  named.  Yes, they are publicly visible, but don't worry about someone
+  walking around the thousands of repos on bitbucket saying "hah, look
+  at the bad coding style of that guy".  Try to get into the mindset
+  that your work is not secret and it's fine that way.  We might not
+  accept it as is for PyPy, asking you instead to improve some things,
+  but we are not going to judge you.
+
+Pull Request
+------------
+
+* The final step is to open a pull request, so that we know that you'd
+  like to merge that branch back to the original ``pypy/pypy`` repo.
+  This can also be done several times if you have interesting
+  intermediate states, but if you get there, then we're likely to
+  proceed to the next stage, which is...
+
+* Get a regular account for pushing directly to
+  ``bitbucket.org/pypy/pypy`` (just ask and you'll get it, basically).
+  Once you have it you can rewrite your file ``.hg/hgrc`` to contain
+  ``default = ssh://hg at bitbucket.org/pypy/pypy``.  Your changes will
+  then be pushed directly to the official repo, but (if you follow these
+  rules) they are still on a branch, and we can still review the
+  branches you want to merge.
+
+* If you get closer to the regular day-to-day development, you'll notice
+  that we generally push small changes as one or a few commits directly
+  to the branch ``default``.  Also, we often collaborate even if we are
+  on other branches, which do not really "belong" to anyone.  At this
+  point you'll need ``hg merge`` and learn how to resolve conflicts that
+  sometimes occur when two people try to push different commits in
+  parallel on the same branch.  But it is likely an issue for later ``:-)``
+
+.. _bitbucket: https://bitbucket.org/
+
+
+Architecture
+^^^^^^^^^^^^
+
+PyPy has layers. Just like ogres or onions. Those layers help us keep the
+respective parts separated enough to be worked on independently and make the
+complexity manageable. This is, again, just a sanity requirement for such
+a complex project. For example writing a new optimization for the JIT usually
+does **not** involve touching a Python interpreter at all or the JIT assembler
+backend or the garbage collector. Instead it requires writing small tests in
+``rpython/jit/metainterp/optimizeopt/test/test_*`` and fixing files there.
+After that, you can just compile PyPy and things should just work.
+
+Further Reading: :doc:`architecture <architecture>`
+
+Where to start?
+---------------
+
+PyPy is made from parts that are relatively independent of each other.
+You should start looking at the part that attracts you most (all paths are
+relative to the PyPy top level directory).  You may look at our
+:doc:`directory reference <dir-reference>` or start off at one of the following
+points:
+
+*  :source:`pypy/interpreter` contains the bytecode interpreter: bytecode dispatcher
+   in :source:`pypy/interpreter/pyopcode.py`, frame and code objects in
+   :source:`pypy/interpreter/eval.py` and :source:`pypy/interpreter/pyframe.py`,
+   function objects and argument passing in :source:`pypy/interpreter/function.py`
+   and :source:`pypy/interpreter/argument.py`, the object space interface
+   definition in :source:`pypy/interpreter/baseobjspace.py`, modules in
+   :source:`pypy/interpreter/module.py` and :source:`pypy/interpreter/mixedmodule.py`.
+   Core types supporting the bytecode interpreter are defined in
+   :source:`pypy/interpreter/typedef.py`.
+
+*  :source:`pypy/interpreter/pyparser` contains a recursive descent parser,
+   and grammar files that allow it to parse the syntax of various Python
+   versions. Once the grammar has been processed, the parser can be
+   translated by the above machinery into efficient code.
+
+*  :source:`pypy/interpreter/astcompiler` contains the compiler.  This
+   contains a modified version of the compiler package from CPython
+   that fixes some bugs and is translatable.
+
+*  :source:`pypy/objspace/std` contains the
+   :ref:`Standard object space <standard-object-space>`.  The main file
+   is :source:`pypy/objspace/std/objspace.py`.  For each type, the file
+   ``xxxobject.py`` contains the implementation for objects of type ``xxx``,
+   as a first approximation.  (Some types have multiple implementations.)
+
+Building
+^^^^^^^^
+
+For building PyPy, we recommend installing a pre-built PyPy first (see
+:doc:`install`). It is possible to build PyPy with CPython, but it will take a
+lot longer to run -- depending on your architecture, between two and three
+times as long.
+
+Further Reading: :doc:`Build <build>`
+
+Coding Guide
+------------
+
+As well as the usual pep8 and formatting standards, there are a number of
+naming conventions and coding styles that are important to understand before
+browsing the source.
+
+Further Reading: :doc:`Coding Guide <coding-guide>`
+
+Testing
+^^^^^^^
+
+Test driven development
+-----------------------
+
+Instead, we practice a lot of test driven development. This is partly because
+of very high quality requirements for compilers and partly because there is
+simply no other way to get around such complex project, that will keep you sane.
+There are probably people out there who are smart enough not to need it, we're
+not one of those. You may consider familiarizing yourself with `pytest`_,
+since this is a tool we use for tests.
+This leads to the next issue:
+
+.. _pytest: http://pytest.org/
+
+py.test and the py lib
+----------------------
+
+The `py.test testing tool`_ drives all our testing needs.
+
+We use the `py library`_ for filesystem path manipulations, terminal
+writing, logging and some other support  functionality.
+
+You don't necessarily need to install these two libraries because
+we also ship them inlined in the PyPy source tree.
+
+.. _py library: http://pylib.readthedocs.org/
+
+Running PyPy's unit tests
+-------------------------
+
+PyPy development always was and is still thoroughly test-driven.
+We use the flexible `py.test testing tool`_ which you can `install independently
+<http://pytest.org/latest/getting-started.html#getstarted>`_ and use for other projects.
+
+The PyPy source tree comes with an inlined version of ``py.test``
+which you can invoke by typing::
+
+    python pytest.py -h
+
+This is usually equivalent to using an installed version::
+
+    py.test -h
+
+If you encounter problems with the installed version
+make sure you have the correct version installed which
+you can find out with the ``--version`` switch.
+
+You will need the `build requirements`_ to run tests successfully, since many of
+them compile little pieces of PyPy and then run the tests inside that minimal
+interpreter. The `cpyext` tests also require `pycparser`, and many tests build
+cases with `hypothesis`.
+
+Now on to running some tests.  PyPy has many different test directories
+and you can use shell completion to point at directories or files::
+
+    py.test pypy/interpreter/test/test_pyframe.py
+
+    # or for running tests of a whole subdirectory
+    py.test pypy/interpreter/
+
+See `py.test usage and invocations`_ for some more generic info
+on how you can run tests.
+
+Beware trying to run "all" pypy tests by pointing to the root
+directory or even the top level subdirectory ``pypy``.  It takes
+hours and uses huge amounts of RAM and is not recommended.
+
+To run CPython regression tests you can point to the ``lib-python``
+directory::
+
+    py.test lib-python/2.7/test/test_datetime.py
+
+This will usually take a long time because this will run
+the PyPy Python interpreter on top of CPython.  On the plus
+side, it's usually still faster than doing a full translation
+and running the regression test with the translated PyPy Python
+interpreter.
+
+.. _py.test testing tool: http://pytest.org
+.. _py.test usage and invocations: http://pytest.org/latest/usage.html#usage
+.. _`build requirements`: build.html#install-build-time-dependencies
+
+Testing After Translation
+^^^^^^^^^^^^^^^^^^^^^^^^^
+
+While the usual invocation of `pytest` translates a piece of RPython code and
+runs it, we have a test extension to run tests without translation, directly
+on the host python. This is very convenient for modules such as `cpyext`, to
+compare and contrast test results between CPython and PyPy. Untranslated tests
+are invoked by using the `-A` or `--runappdirect` option to `pytest`::
+
+    python2 pytest.py -A pypy/module/cpyext/test
+
+where `python2` can be either `python2` or `pypy2`. On the `py3` branch, the
+collection phase must be run with `python2` so untranslated tests are run
+with::
+
+    cpython2 pytest.py -A pypy/module/cpyext/test --python=path/to/pypy3
+
+
+Tooling & Utilities
+^^^^^^^^^^^^^^^^^^^
+
+If you are interested in the inner workings of the PyPy Python interpreter,
+there are some features of the untranslated Python interpreter that allow you
+to introspect its internals.
+
+
+Interpreter-level console
+-------------------------
+
+To start interpreting Python with PyPy, install a C compiler that is
+supported by distutils and use Python 2.7 or greater to run PyPy::
+
+    cd pypy
+    python bin/pyinteractive.py
+
+After a few seconds (remember: this is running on top of CPython), you should
+be at the PyPy prompt, which is the same as the Python prompt, but with an
+extra ">".
+
+If you press
+<Ctrl-C> on the console you enter the interpreter-level console, a
+usual CPython console.  You can then access internal objects of PyPy
+(e.g. the :ref:`object space <objspace>`) and any variables you have created on the PyPy
+prompt with the prefix ``w_``::
+
+    >>>> a = 123
+    >>>> <Ctrl-C>
+    *** Entering interpreter-level console ***
+    >>> w_a
+    W_IntObject(123)
+
+The mechanism works in both directions. If you define a variable with the ``w_`` prefix on the interpreter-level, you will see it on the app-level::
+
+    >>> w_l = space.newlist([space.wrap(1), space.wrap("abc")])
+    >>> <Ctrl-D>
+    *** Leaving interpreter-level console ***
+
+    KeyboardInterrupt
+    >>>> l
+    [1, 'abc']
+
+Note that the prompt of the interpreter-level console is only '>>>' since
+it runs on CPython level. If you want to return to PyPy, press <Ctrl-D> (under
+Linux) or <Ctrl-Z>, <Enter> (under Windows).
+
+Also note that not all modules are available by default in this mode (for
+example: ``_continuation`` needed by ``greenlet``) , you may need to use one of
+``--withmod-...`` command line options.
+
+You may be interested in reading more about the distinction between
+:ref:`interpreter-level and app-level <interpreter-level>`.
+
+pyinteractive.py options
+------------------------
+
+To list the PyPy interpreter command line options, type::
+
+    cd pypy
+    python bin/pyinteractive.py --help
+
+pyinteractive.py supports most of the options that CPython supports too (in addition to a
+large amount of options that can be used to customize pyinteractive.py).
+As an example of using PyPy from the command line, you could type::
+
+    python pyinteractive.py --withmod-time -c "from test import pystone; pystone.main(10)"
+
+Alternatively, as with regular Python, you can simply give a
+script name on the command line::
+
+    python pyinteractive.py --withmod-time ../../lib-python/2.7/test/pystone.py 10
+
+The ``--withmod-xxx`` option enables the built-in module ``xxx``.  By
+default almost none of them are, because initializing them takes time.
+If you want anyway to enable all built-in modules, you can use
+``--allworkingmodules``.
+
+See our :doc:`configuration sections <config/index>` for details about what all the commandline
+options do.
+
+
+.. _trace example:
+
+Tracing bytecode and operations on objects
+------------------------------------------
+
+You can use a simple tracing mode to monitor the interpretation of
+bytecodes.  To enable it, set ``__pytrace__ = 1`` on the interactive
+PyPy console::
+
+    >>>> __pytrace__ = 1
+    Tracing enabled
+    >>>> x = 5
+            <module>:           LOAD_CONST    0 (5)
+            <module>:           STORE_NAME    0 (x)
+            <module>:           LOAD_CONST    1 (None)
+            <module>:           RETURN_VALUE    0
+    >>>> x
+            <module>:           LOAD_NAME    0 (x)
+            <module>:           PRINT_EXPR    0
+    5
+            <module>:           LOAD_CONST    0 (None)
+            <module>:           RETURN_VALUE    0
+    >>>>
+
+
+Demos
+^^^^^
+
+The `example-interpreter`_ repository contains an example interpreter
+written using the RPython translation toolchain.
+
+.. _example-interpreter: https://bitbucket.org/pypy/example-interpreter
+
+
+graphviz & pygame for flow graph viewing (highly recommended)
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+graphviz and pygame are both necessary if you want to look at generated flow
+graphs:
+
+    graphviz: http://www.graphviz.org/Download.php
+
+    pygame: http://www.pygame.org/download.shtml
+
diff --git a/pypy/doc/contributor.rst b/pypy/doc/contributor.rst
--- a/pypy/doc/contributor.rst
+++ b/pypy/doc/contributor.rst
@@ -214,6 +214,7 @@
   Lukas Vacek
   Omer Katz
   Jacek Generowicz
+  Tomasz Dziopa
   Sylvain Thenault
   Jakub Stasiak
   Andrew Dalke
@@ -274,6 +275,7 @@
   Yury V. Zaytsev
   florinpapa
   Anders Sigfridsson
+  Matt Jackson
   Nikolay Zinov
   rafalgalczynski at gmail.com
   Joshua Gilbert
diff --git a/pypy/doc/discussion/ctypes-implementation.rst b/pypy/doc/discussion/ctypes-implementation.rst
--- a/pypy/doc/discussion/ctypes-implementation.rst
+++ b/pypy/doc/discussion/ctypes-implementation.rst
@@ -141,28 +141,3 @@
 
 .. _pyglet: http://pyglet.org/

From pypy.commits at gmail.com  Sun Jul 29 08:10:59 2018
From: pypy.commits at gmail.com (daa)
Date: Sun, 29 Jul 2018 05:10:59 -0700 (PDT)
Subject: [pypy-commit] cffi default: always specify entry points because
 they may be used even with PyPy
Message-ID: <5b5daed3.1c69fb81.4ef7f.1c19@mx.google.com>

Author: daa <aduryagin at gmail.com>
Branch: 
Changeset: r3130:64dbf3771c3c
Date: 2018-07-29 01:39 +0300
http://bitbucket.org/cffi/cffi/changeset/64dbf3771c3c/

Log:	always specify entry points because they may be used even with PyPy

diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -230,7 +230,7 @@
             "distutils.setup_keywords": [
                 "cffi_modules = cffi.setuptools_ext:cffi_modules",
             ],
-        } if cpython else {},
+        },
 
         classifiers=[
             'Programming Language :: Python',

From pypy.commits at gmail.com  Sun Jul 29 11:29:55 2018
From: pypy.commits at gmail.com (arigo)
Date: Sun, 29 Jul 2018 08:29:55 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6: Merged in
 jKorvin/pypy/fix-float-deprecation-warning (pull request #618)
Message-ID: <5b5ddd73.1c69fb81.6faec.1537@mx.google.com>

Author: Armin Rigo <armin.rigo at gmail.com>
Branch: py3.6
Changeset: r94928:b72ee387ca84
Date: 2018-07-29 15:29 +0000
http://bitbucket.org/pypy/pypy/changeset/b72ee387ca84/

Log:	Merged in jKorvin/pypy/fix-float-deprecation-warning (pull request
	#618)

	Add deprecation warning if type of result of __float__ is float
	inherit class

diff --git a/pypy/objspace/std/floatobject.py b/pypy/objspace/std/floatobject.py
--- a/pypy/objspace/std/floatobject.py
+++ b/pypy/objspace/std/floatobject.py
@@ -220,6 +220,15 @@
         if space.lookup(w_value, "__float__") is not None:
             w_obj = space.float(w_value)
             if space.is_w(w_floattype, space.w_float):
+                w_obj_type = space.type(w_obj)
+                if not space.is_w(w_obj_type, space.w_float):
+                    space.warn(space.newtext(
+                        "%s.__float__ returned non-float (type %s).  "
+                        "The ability to return an instance of a strict subclass "
+                        "of float is deprecated, and may be removed "
+                        "in a future version of Python." %
+                            (space.type(w_value).name, w_obj_type.name)),
+                        space.w_DeprecationWarning)
                 return w_obj
             value = space.float_w(w_obj)
         elif space.isinstance_w(w_value, space.w_unicode):
diff --git a/pypy/objspace/std/test/test_floatobject.py b/pypy/objspace/std/test/test_floatobject.py
--- a/pypy/objspace/std/test/test_floatobject.py
+++ b/pypy/objspace/std/test/test_floatobject.py
@@ -299,6 +299,21 @@
                 return 42.
         assert float(X()) == 42.
 
+    def test_float_conversion_deprecated_warning(self):
+        import warnings
+
+        class X(float):
+            def __float__(self):
+                return self
+        x = X(42)
+
+        with warnings.catch_warnings(record=True) as log:
+            warnings.simplefilter("always", DeprecationWarning)
+            converted_x = float(x)
+
+        assert converted_x == 42.  # sanity check
+        assert len(log) == 1
+
     def test_round(self):
         import math
         assert 1.0 == round(1.0)

From pypy.commits at gmail.com  Sun Jul 29 11:30:10 2018
From: pypy.commits at gmail.com (Evgenii Gorinov)
Date: Sun, 29 Jul 2018 08:30:10 -0700 (PDT)
Subject: [pypy-commit] pypy fix-float-deprecation-warning: Add deprecation
 warning if type of result of __float__ is float inherit class
Message-ID: <5b5ddd82.1c69fb81.9168b.b351@mx.google.com>

Author: Evgenii Gorinov <jonny-k at yandex-team.ru>
Branch: fix-float-deprecation-warning
Changeset: r94927:817e823be049
Date: 2018-07-29 14:45 +0100
http://bitbucket.org/pypy/pypy/changeset/817e823be049/

Log:	Add deprecation warning if type of result of __float__ is float
	inherit class

diff --git a/pypy/objspace/std/floatobject.py b/pypy/objspace/std/floatobject.py
--- a/pypy/objspace/std/floatobject.py
+++ b/pypy/objspace/std/floatobject.py
@@ -220,6 +220,15 @@
         if space.lookup(w_value, "__float__") is not None:
             w_obj = space.float(w_value)
             if space.is_w(w_floattype, space.w_float):
+                w_obj_type = space.type(w_obj)
+                if not space.is_w(w_obj_type, space.w_float):
+                    space.warn(space.newtext(
+                        "%s.__float__ returned non-float (type %s).  "
+                        "The ability to return an instance of a strict subclass "
+                        "of float is deprecated, and may be removed "
+                        "in a future version of Python." %
+                            (space.type(w_value).name, w_obj_type.name)),
+                        space.w_DeprecationWarning)
                 return w_obj
             value = space.float_w(w_obj)
         elif space.isinstance_w(w_value, space.w_unicode):
diff --git a/pypy/objspace/std/test/test_floatobject.py b/pypy/objspace/std/test/test_floatobject.py
--- a/pypy/objspace/std/test/test_floatobject.py
+++ b/pypy/objspace/std/test/test_floatobject.py
@@ -299,6 +299,21 @@
                 return 42.
         assert float(X()) == 42.
 
+    def test_float_conversion_deprecated_warning(self):
+        import warnings
+
+        class X(float):
+            def __float__(self):
+                return self
+        x = X(42)
+
+        with warnings.catch_warnings(record=True) as log:
+            warnings.simplefilter("always", DeprecationWarning)
+            converted_x = float(x)
+
+        assert converted_x == 42.  # sanity check
+        assert len(log) == 1
+
     def test_round(self):
         import math
         assert 1.0 == round(1.0)

From pypy.commits at gmail.com  Sun Jul 29 11:44:15 2018
From: pypy.commits at gmail.com (arigo)
Date: Sun, 29 Jul 2018 08:44:15 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6: Merged in raduciorba/pypy/py3.6 (pull
 request #619)
Message-ID: <5b5de0cf.1c69fb81.30951.05e9@mx.google.com>

Author: Armin Rigo <armin.rigo at gmail.com>
Branch: py3.6
Changeset: r94931:77bcaf8bbc5f
Date: 2018-07-29 15:43 +0000
http://bitbucket.org/pypy/pypy/changeset/77bcaf8bbc5f/

Log:	Merged in raduciorba/pypy/py3.6 (pull request #619)

	fix async generator bug when yielding a StopIteration

diff --git a/pypy/interpreter/generator.py b/pypy/interpreter/generator.py
--- a/pypy/interpreter/generator.py
+++ b/pypy/interpreter/generator.py
@@ -199,7 +199,6 @@
         except OperationError as e:
             if not e.match(space, space.w_StopIteration):
                 raise
-            e.normalize_exception(space)
             frame._report_stopiteration_sometimes(w_yf, e)
             try:
                 w_stop_value = space.getattr(e.get_w_value(space),
@@ -679,11 +678,8 @@
 
     def unwrap_value(self, w_value):
         if isinstance(w_value, AsyncGenValueWrapper):
-            w_value = w_value.w_value
-            if self.space.isinstance_w(w_value, self.space.w_tuple):
-                # Construct the exception manually, to avoid tuple unpacking.
-                w_value = self.space.call_function(self.space.w_StopIteration,
-                                                   w_value)
+            w_value = self.space.call_function(self.space.w_StopIteration,
+                                               w_value.w_value)
             raise OperationError(self.space.w_StopIteration, w_value)
         else:
             return w_value
diff --git a/pypy/interpreter/test/test_coroutine.py b/pypy/interpreter/test/test_coroutine.py
--- a/pypy/interpreter/test/test_coroutine.py
+++ b/pypy/interpreter/test/test_coroutine.py
@@ -671,3 +671,20 @@
 
         assert self.run_async(run()) == ([], (1,))
         """
+
+    def test_asyncgen_yield_stopiteration(self):
+        """
+        async def foo():
+            yield 1
+            yield StopIteration(2)
+
+        async def run():
+            it = foo().__aiter__()
+            val1 = await it.__anext__()
+            assert val1 == 1
+            val2 = await it.__anext__()
+            assert isinstance(val2, StopIteration)
+            assert val2.value == 2
+
+        self.run_async(run())
+        """

From pypy.commits at gmail.com  Sun Jul 29 11:44:26 2018
From: pypy.commits at gmail.com (raduciorba)
Date: Sun, 29 Jul 2018 08:44:26 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6: add failing test for yielding a
 StopIteration from an asyncgen
Message-ID: <5b5de0da.1c69fb81.d14e3.974d@mx.google.com>

Author: Radu Ciorba <radu at devrandom.ro>
Branch: py3.6
Changeset: r94929:d890ba3db546
Date: 2018-07-29 16:49 +0300
http://bitbucket.org/pypy/pypy/changeset/d890ba3db546/

Log:	add failing test for yielding a StopIteration from an asyncgen

diff --git a/pypy/interpreter/test/test_coroutine.py b/pypy/interpreter/test/test_coroutine.py
--- a/pypy/interpreter/test/test_coroutine.py
+++ b/pypy/interpreter/test/test_coroutine.py
@@ -671,3 +671,20 @@
 
         assert self.run_async(run()) == ([], (1,))
         """
+
+    def test_asyncgen_yield_stopiteration(self):
+        """
+        async def foo():
+            yield 1
+            yield StopIteration(2)
+
+        async def run():
+            it = foo().__aiter__()
+            val1 = await it.__anext__()
+            assert val1 == 1
+            val2 = await it.__anext__()
+            assert isinstance(val2, StopIteration)
+            assert val2.value == 2
+
+        self.run_async(run())
+        """

From pypy.commits at gmail.com  Sun Jul 29 11:44:28 2018
From: pypy.commits at gmail.com (raduciorba)
Date: Sun, 29 Jul 2018 08:44:28 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6: fix async generator bug when yielding a
 StopIteration
Message-ID: <5b5de0dc.1c69fb81.64668.319c@mx.google.com>

Author: Radu Ciorba <radu at devrandom.ro>
Branch: py3.6
Changeset: r94930:aac246060deb
Date: 2018-07-29 16:27 +0300
http://bitbucket.org/pypy/pypy/changeset/aac246060deb/

Log:	fix async generator bug when yielding a StopIteration

diff --git a/pypy/interpreter/generator.py b/pypy/interpreter/generator.py
--- a/pypy/interpreter/generator.py
+++ b/pypy/interpreter/generator.py
@@ -199,7 +199,6 @@
         except OperationError as e:
             if not e.match(space, space.w_StopIteration):
                 raise
-            e.normalize_exception(space)
             frame._report_stopiteration_sometimes(w_yf, e)
             try:
                 w_stop_value = space.getattr(e.get_w_value(space),
@@ -679,11 +678,8 @@
 
     def unwrap_value(self, w_value):
         if isinstance(w_value, AsyncGenValueWrapper):
-            w_value = w_value.w_value
-            if self.space.isinstance_w(w_value, self.space.w_tuple):
-                # Construct the exception manually, to avoid tuple unpacking.
-                w_value = self.space.call_function(self.space.w_StopIteration,
-                                                   w_value)
+            w_value = self.space.call_function(self.space.w_StopIteration,
+                                               w_value.w_value)
             raise OperationError(self.space.w_StopIteration, w_value)
         else:
             return w_value